From 2bb0fdc6237ce6b024fb21dc1c1ebe54a4247d41 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Thu, 28 Sep 2023 06:20:22 -0700 Subject: [PATCH 01/89] Bump to SP 1.0.2 release (#4151) --- tools/hubris_checksums | 14 +++++++------- tools/hubris_version | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/hubris_checksums b/tools/hubris_checksums index c8d24b5dac..e2b16d345e 100644 --- a/tools/hubris_checksums +++ b/tools/hubris_checksums @@ -1,7 +1,7 @@ -36ea4c454e78ecdad926bf321493fc28ff79e3133aa67ef4fd355b8c7f92676a build-gimlet-c-image-default-v1.0.1.zip -61f387726131e9ff98b8645034144f421c1d7bd1466b35584fd93901b6c5d5ae build-gimlet-d-image-default-v1.0.1.zip -63c986402aab5b3aef1f04e958234f395f40a65d218812c22a154695818e1303 build-gimlet-e-image-default-v1.0.1.zip -526442e6d4466e357aa7f69fed400693eb542f239a03f4fc4d4c74d74afcbd6d build-psc-b-image-default-v1.0.1.zip -91d8537e4a5524e2d8a8047179b8eed11ca22fd741dea27edade921efd980aa2 build-psc-c-image-default-v1.0.1.zip -50932dcb7da2c89b6958acfa36972f2d32159bb59489dc335028c49ae60702c5 build-sidecar-b-image-default-v1.0.1.zip -d466495e45101aa21ad9cfb136bb5f1f2288bb0285e1e6f5b257d21ef5c53b86 build-sidecar-c-image-default-v1.0.1.zip +204328b941deab8bfd3d6e34af96ef04489672fa3b0d5419b60456f9b052e789 build-gimlet-c-image-default-v1.0.2.zip +0ebaa9d98c3734420a482160a7a19dd09510ea3bdc573a090a97ec47137bd624 build-gimlet-d-image-default-v1.0.2.zip +39ec8fd0c946b744e59d9c1b89914f354c60f54e974398029d1dea9d31681f05 build-gimlet-e-image-default-v1.0.2.zip +fa5dc36a7a70eeb45d4c4b3b314ba54ee820b3d57ffc07defcc3ae07c142329c build-psc-b-image-default-v1.0.2.zip +4a9850100f8b5fcbbdd11d41ccd8d5037059697a9b65c1ba2dba48a6565ba9e7 build-psc-c-image-default-v1.0.2.zip +1bb870d7921c1731ec83dc38b8e3425703ec17efa614d75e92f07a551312f54b build-sidecar-b-image-default-v1.0.2.zip +6aed0e15e0025bb87a22ecea60d75fa71b54b83bea1e213b8cd5bdb02e7ccb2d build-sidecar-c-image-default-v1.0.2.zip diff --git a/tools/hubris_version b/tools/hubris_version index 3c08af7387..0f1729d791 100644 --- a/tools/hubris_version +++ b/tools/hubris_version @@ -1 +1 @@ -TAGS=(gimlet-v1.0.1 psc-v1.0.1 sidecar-v1.0.1) +TAGS=(gimlet-v1.0.2 psc-v1.0.2 sidecar-v1.0.2) From 8602e4632abf9065bf86c387c246cd3534b81f22 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 28 Sep 2023 13:20:58 -0700 Subject: [PATCH 02/89] rust_toolchain: 1.72.0 -> 1.72.1 (#4159) --- .github/buildomat/jobs/build-and-test-helios.sh | 2 +- .github/buildomat/jobs/build-and-test-linux.sh | 2 +- .github/buildomat/jobs/ci-tools.sh | 2 +- .github/buildomat/jobs/clippy.sh | 2 +- .github/buildomat/jobs/host-image.sh | 2 +- .github/buildomat/jobs/package.sh | 2 +- .github/buildomat/jobs/trampoline-image.sh | 2 +- rust-toolchain.toml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/buildomat/jobs/build-and-test-helios.sh b/.github/buildomat/jobs/build-and-test-helios.sh index c1a9bee581..f9722a2b92 100755 --- a/.github/buildomat/jobs/build-and-test-helios.sh +++ b/.github/buildomat/jobs/build-and-test-helios.sh @@ -3,7 +3,7 @@ #: name = "build-and-test (helios)" #: variety = "basic" #: target = "helios-2.0" -#: rust_toolchain = "1.72.0" +#: rust_toolchain = "1.72.1" #: output_rules = [ #: "/var/tmp/omicron_tmp/*", #: "!/var/tmp/omicron_tmp/crdb-base*", diff --git a/.github/buildomat/jobs/build-and-test-linux.sh b/.github/buildomat/jobs/build-and-test-linux.sh index 50bdb22e56..f33d1a8cfa 100755 --- a/.github/buildomat/jobs/build-and-test-linux.sh +++ b/.github/buildomat/jobs/build-and-test-linux.sh @@ -3,7 +3,7 @@ #: name = "build-and-test (ubuntu-20.04)" #: variety = "basic" #: target = "ubuntu-20.04" -#: rust_toolchain = "1.72.0" +#: rust_toolchain = "1.72.1" #: output_rules = [ #: "/var/tmp/omicron_tmp/*", #: "!/var/tmp/omicron_tmp/crdb-base*", diff --git a/.github/buildomat/jobs/ci-tools.sh b/.github/buildomat/jobs/ci-tools.sh index b5d1d26f1a..702561a951 100755 --- a/.github/buildomat/jobs/ci-tools.sh +++ b/.github/buildomat/jobs/ci-tools.sh @@ -3,7 +3,7 @@ #: name = "helios / CI tools" #: variety = "basic" #: target = "helios-2.0" -#: rust_toolchain = "1.72.0" +#: rust_toolchain = "1.72.1" #: output_rules = [ #: "=/work/end-to-end-tests/*.gz", #: "=/work/caboose-util.gz", diff --git a/.github/buildomat/jobs/clippy.sh b/.github/buildomat/jobs/clippy.sh index 184b4ccbe6..dba1021919 100755 --- a/.github/buildomat/jobs/clippy.sh +++ b/.github/buildomat/jobs/clippy.sh @@ -3,7 +3,7 @@ #: name = "clippy (helios)" #: variety = "basic" #: target = "helios-2.0" -#: rust_toolchain = "1.72.0" +#: rust_toolchain = "1.72.1" #: output_rules = [] # Run clippy on illumos (not just other systems) because a bunch of our code diff --git a/.github/buildomat/jobs/host-image.sh b/.github/buildomat/jobs/host-image.sh index 0e2191ebc8..ba0b4e1ac3 100755 --- a/.github/buildomat/jobs/host-image.sh +++ b/.github/buildomat/jobs/host-image.sh @@ -3,7 +3,7 @@ #: name = "helios / build OS image" #: variety = "basic" #: target = "helios-2.0" -#: rust_toolchain = "1.72.0" +#: rust_toolchain = "1.72.1" #: output_rules = [ #: "=/work/helios/image/output/os.tar.gz", #: ] diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index eedb3b1723..fe5d6a9b7f 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -3,7 +3,7 @@ #: name = "helios / package" #: variety = "basic" #: target = "helios-2.0" -#: rust_toolchain = "1.72.0" +#: rust_toolchain = "1.72.1" #: output_rules = [ #: "=/work/version.txt", #: "=/work/package.tar.gz", diff --git a/.github/buildomat/jobs/trampoline-image.sh b/.github/buildomat/jobs/trampoline-image.sh index 794ebd96c1..6014d7dca0 100755 --- a/.github/buildomat/jobs/trampoline-image.sh +++ b/.github/buildomat/jobs/trampoline-image.sh @@ -3,7 +3,7 @@ #: name = "helios / build trampoline OS image" #: variety = "basic" #: target = "helios-2.0" -#: rust_toolchain = "1.72.0" +#: rust_toolchain = "1.72.1" #: output_rules = [ #: "=/work/helios/image/output/os.tar.gz", #: ] diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 77379db470..eacfef491b 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -4,5 +4,5 @@ # # We choose a specific toolchain (rather than "stable") for repeatability. The # intent is to keep this up-to-date with recently-released stable Rust. -channel = "1.72.0" +channel = "1.72.1" profile = "default" From 8fb68a25e4ccf34625ee3aaf032f5a013ada8292 Mon Sep 17 00:00:00 2001 From: Rain Date: Thu, 28 Sep 2023 16:32:53 -0700 Subject: [PATCH 03/89] [meta] add a workspace-hack package managed by hakari (#4154) Similar to https://github.com/oxidecomputer/crucible/pull/956, this adds a workspace-hack package managed by cargo-hakari. This should massively cut down on incremental build times. `cargo hakari` doesn't yet support writing out `workspace-hack` dependency lines using workspace inheritance (https://github.com/guppy-rs/guppy/issues/7), so skip over it in the lint. Reverting this change can be done by following the instructions at https://docs.rs/cargo-hakari/latest/cargo_hakari/#disabling-and-uninstalling. --- .config/hakari.toml | 29 +++ .github/workflows/hakari.yml | 37 ++++ Cargo.lock | 169 +++++++++++++++ Cargo.toml | 1 + README.adoc | 29 +-- api_identity/Cargo.toml | 1 + bootstore/Cargo.toml | 1 + bootstrap-agent-client/Cargo.toml | 1 + caboose-util/Cargo.toml | 1 + certificates/Cargo.toml | 1 + common/Cargo.toml | 1 + crdb-seed/Cargo.toml | 1 + ddm-admin-client/Cargo.toml | 1 + deploy/Cargo.toml | 1 + dev-tools/omdb/Cargo.toml | 1 + dev-tools/omicron-dev/Cargo.toml | 1 + dev-tools/xtask/src/main.rs | 14 ++ dns-server/Cargo.toml | 1 + dns-service-client/Cargo.toml | 1 + dpd-client/Cargo.toml | 1 + end-to-end-tests/Cargo.toml | 1 + gateway-cli/Cargo.toml | 1 + gateway-client/Cargo.toml | 1 + gateway-test-utils/Cargo.toml | 1 + gateway/Cargo.toml | 1 + illumos-utils/Cargo.toml | 1 + installinator-artifact-client/Cargo.toml | 1 + installinator-artifactd/Cargo.toml | 1 + installinator-common/Cargo.toml | 1 + installinator/Cargo.toml | 1 + internal-dns-cli/Cargo.toml | 1 + internal-dns/Cargo.toml | 1 + ipcc-key-value/Cargo.toml | 1 + key-manager/Cargo.toml | 1 + nexus-client/Cargo.toml | 1 + nexus/Cargo.toml | 1 + nexus/authz-macros/Cargo.toml | 1 + nexus/db-macros/Cargo.toml | 1 + nexus/db-model/Cargo.toml | 1 + nexus/db-queries/Cargo.toml | 1 + nexus/defaults/Cargo.toml | 1 + nexus/test-interface/Cargo.toml | 1 + nexus/test-utils-macros/Cargo.toml | 1 + nexus/test-utils/Cargo.toml | 1 + nexus/types/Cargo.toml | 1 + oxide-client/Cargo.toml | 1 + oximeter-client/Cargo.toml | 1 + oximeter/collector/Cargo.toml | 1 + oximeter/db/Cargo.toml | 1 + oximeter/instruments/Cargo.toml | 1 + oximeter/oximeter-macro-impl/Cargo.toml | 1 + oximeter/oximeter/Cargo.toml | 1 + oximeter/producer/Cargo.toml | 1 + package/Cargo.toml | 1 + passwords/Cargo.toml | 1 + rpaths/Cargo.toml | 1 + sled-agent-client/Cargo.toml | 1 + sled-agent/Cargo.toml | 1 + sled-hardware/Cargo.toml | 1 + sp-sim/Cargo.toml | 1 + test-utils/Cargo.toml | 1 + tufaceous-lib/Cargo.toml | 1 + tufaceous/Cargo.toml | 1 + update-engine/Cargo.toml | 1 + wicket-common/Cargo.toml | 1 + wicket-dbg/Cargo.toml | 1 + wicket/Cargo.toml | 1 + wicketd-client/Cargo.toml | 1 + wicketd/Cargo.toml | 1 + workspace-hack/.gitattributes | 4 + workspace-hack/Cargo.toml | 261 +++++++++++++++++++++++ workspace-hack/build.rs | 2 + workspace-hack/src/lib.rs | 1 + 73 files changed, 589 insertions(+), 21 deletions(-) create mode 100644 .config/hakari.toml create mode 100644 .github/workflows/hakari.yml create mode 100644 workspace-hack/.gitattributes create mode 100644 workspace-hack/Cargo.toml create mode 100644 workspace-hack/build.rs create mode 100644 workspace-hack/src/lib.rs diff --git a/.config/hakari.toml b/.config/hakari.toml new file mode 100644 index 0000000000..9562f92300 --- /dev/null +++ b/.config/hakari.toml @@ -0,0 +1,29 @@ +# This file contains settings for `cargo hakari`. +# See https://docs.rs/cargo-hakari/latest/cargo_hakari/config for a full list of options. + +hakari-package = "omicron-workspace-hack" + +# Format for `workspace-hack = ...` lines in other Cargo.tomls. Requires cargo-hakari 0.9.8 or above. +dep-format-version = "4" + +# Setting workspace.resolver = "2" in the root Cargo.toml is HIGHLY recommended. +# Hakari works much better with the new feature resolver. +# For more about the new feature resolver, see: +# https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver +resolver = "2" + +# Add triples corresponding to platforms commonly used by developers here. +# https://doc.rust-lang.org/rustc/platform-support.html +platforms = [ + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-unknown-illumos", + # "x86_64-pc-windows-msvc", +] + +[traversal-excludes] +workspace-members = ["xtask"] + +# Write out exact versions rather than a semver range. (Defaults to false.) +# exact-versions = true diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml new file mode 100644 index 0000000000..d79196d318 --- /dev/null +++ b/.github/workflows/hakari.yml @@ -0,0 +1,37 @@ +# This workflow file serves as an example for cargo-hakari CI integration. + +on: + push: + branches: + - main + pull_request: + branches: + - main + +name: cargo hakari + +jobs: + workspace-hack-check: + name: Check workspace-hack + runs-on: ubuntu-latest + env: + RUSTFLAGS: -D warnings + steps: + - uses: actions/checkout@v4 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: Install cargo-hakari + uses: taiki-e/install-action@v2 + with: + tool: cargo-hakari + - name: Check workspace-hack Cargo.toml is up-to-date + uses: actions-rs/cargo@v1 + with: + command: hakari + args: generate --diff + - name: Check all crates depend on workspace-hack + uses: actions-rs/cargo@v1 + with: + command: hakari + args: manage-deps --dry-run diff --git a/Cargo.lock b/Cargo.lock index e5130b6b33..5f073db250 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,6 +184,7 @@ dependencies = [ name = "api_identity" version = "0.1.0" dependencies = [ + "omicron-workspace-hack", "proc-macro2", "quote", "syn 2.0.32", @@ -370,6 +371,7 @@ name = "authz-macros" version = "0.1.0" dependencies = [ "heck 0.4.1", + "omicron-workspace-hack", "proc-macro2", "quote", "serde", @@ -658,6 +660,7 @@ dependencies = [ "omicron-common 0.1.0", "omicron-rpaths", "omicron-test-utils", + "omicron-workspace-hack", "pq-sys", "proptest", "rand 0.8.5", @@ -684,6 +687,7 @@ dependencies = [ "chrono", "ipnetwork", "omicron-common 0.1.0", + "omicron-workspace-hack", "progenitor", "regress", "reqwest", @@ -789,6 +793,7 @@ version = "0.1.0" dependencies = [ "anyhow", "hubtools", + "omicron-workspace-hack", ] [[package]] @@ -1245,6 +1250,7 @@ dependencies = [ "dropshot", "hex", "omicron-test-utils", + "omicron-workspace-hack", "ring", "slog", "tokio", @@ -1726,6 +1732,7 @@ name = "db-macros" version = "0.1.0" dependencies = [ "heck 0.4.1", + "omicron-workspace-hack", "proc-macro2", "quote", "rustfmt-wrapper", @@ -1741,6 +1748,7 @@ dependencies = [ "anyhow", "either", "omicron-common 0.1.0", + "omicron-workspace-hack", "omicron-zone-package", "progenitor", "progenitor-client", @@ -2009,6 +2017,7 @@ dependencies = [ "expectorate", "http", "omicron-test-utils", + "omicron-workspace-hack", "openapi-lint", "openapiv3", "pretty-hex 0.3.0", @@ -2039,6 +2048,7 @@ version = "0.1.0" dependencies = [ "chrono", "http", + "omicron-workspace-hack", "progenitor", "reqwest", "schemars", @@ -2095,6 +2105,7 @@ dependencies = [ "futures", "http", "ipnetwork", + "omicron-workspace-hack", "omicron-zone-package", "progenitor", "progenitor-client", @@ -2280,6 +2291,7 @@ dependencies = [ "http", "omicron-sled-agent", "omicron-test-utils", + "omicron-workspace-hack", "oxide-client", "rand 0.8.5", "reqwest", @@ -2720,6 +2732,7 @@ dependencies = [ "hex", "libc", "omicron-common 0.1.0", + "omicron-workspace-hack", "reqwest", "serde", "serde_json", @@ -2738,6 +2751,7 @@ version = "0.1.0" dependencies = [ "base64 0.21.4", "chrono", + "omicron-workspace-hack", "progenitor", "rand 0.8.5", "reqwest", @@ -2802,6 +2816,7 @@ dependencies = [ "gateway-messages", "omicron-gateway", "omicron-test-utils", + "omicron-workspace-hack", "slog", "sp-sim", "tokio", @@ -3379,6 +3394,7 @@ dependencies = [ "macaddr", "mockall", "omicron-common 0.1.0", + "omicron-workspace-hack", "opte-ioctl", "oxide-vpc", "regress", @@ -3488,6 +3504,7 @@ dependencies = [ "libc", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "once_cell", "partial-io", "progenitor-client", @@ -3517,6 +3534,7 @@ name = "installinator-artifact-client" version = "0.1.0" dependencies = [ "installinator-common", + "omicron-workspace-hack", "progenitor", "regress", "reqwest", @@ -3542,6 +3560,7 @@ dependencies = [ "installinator-common", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "openapi-lint", "openapiv3", "schemars", @@ -3561,6 +3580,7 @@ dependencies = [ "camino", "illumos-utils", "omicron-common 0.1.0", + "omicron-workspace-hack", "schemars", "serde", "serde_json", @@ -3593,6 +3613,7 @@ dependencies = [ "hyper", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "progenitor", "reqwest", "serde", @@ -3635,6 +3656,7 @@ dependencies = [ "dropshot", "internal-dns 0.1.0", "omicron-common 0.1.0", + "omicron-workspace-hack", "slog", "tokio", "trust-dns-resolver", @@ -3658,6 +3680,7 @@ dependencies = [ "ciborium", "libc", "omicron-common 0.1.0", + "omicron-workspace-hack", "proptest", "serde", "test-strategy", @@ -3762,6 +3785,7 @@ dependencies = [ "async-trait", "hkdf", "omicron-common 0.1.0", + "omicron-workspace-hack", "secrecy", "sha3", "slog", @@ -4252,6 +4276,7 @@ dependencies = [ "ipnetwork", "omicron-common 0.1.0", "omicron-passwords 0.1.0", + "omicron-workspace-hack", "progenitor", "regress", "reqwest", @@ -4301,6 +4326,7 @@ dependencies = [ "omicron-common 0.1.0", "omicron-passwords 0.1.0", "omicron-rpaths", + "omicron-workspace-hack", "parse-display", "pq-sys", "rand 0.8.5", @@ -4356,6 +4382,7 @@ dependencies = [ "omicron-rpaths", "omicron-sled-agent", "omicron-test-utils", + "omicron-workspace-hack", "once_cell", "openapiv3", "openssl", @@ -4401,6 +4428,7 @@ dependencies = [ "ipnetwork", "lazy_static", "omicron-common 0.1.0", + "omicron-workspace-hack", "rand 0.8.5", "serde_json", ] @@ -4414,6 +4442,7 @@ dependencies = [ "internal-dns 0.1.0", "nexus-types", "omicron-common 0.1.0", + "omicron-workspace-hack", "slog", "uuid", ] @@ -4442,6 +4471,7 @@ dependencies = [ "omicron-passwords 0.1.0", "omicron-sled-agent", "omicron-test-utils", + "omicron-workspace-hack", "oximeter 0.1.0", "oximeter-client", "oximeter-collector", @@ -4461,6 +4491,7 @@ dependencies = [ name = "nexus-test-utils-macros" version = "0.1.0" dependencies = [ + "omicron-workspace-hack", "proc-macro2", "quote", "syn 2.0.32", @@ -4479,6 +4510,7 @@ dependencies = [ "newtype_derive", "omicron-common 0.1.0", "omicron-passwords 0.1.0", + "omicron-workspace-hack", "openssl", "openssl-probe", "openssl-sys", @@ -4785,6 +4817,7 @@ dependencies = [ "foreign-types 0.3.2", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "openssl", "openssl-sys", "rcgen", @@ -4812,6 +4845,7 @@ dependencies = [ "lazy_static", "libc", "macaddr", + "omicron-workspace-hack", "parse-display", "progenitor", "proptest", @@ -4885,6 +4919,7 @@ dependencies = [ "clap 4.4.3", "crossbeam", "omicron-package", + "omicron-workspace-hack", "serde", "serde_derive", "thiserror", @@ -4910,6 +4945,7 @@ dependencies = [ "omicron-rpaths", "omicron-sled-agent", "omicron-test-utils", + "omicron-workspace-hack", "openssl", "oxide-client", "pq-sys", @@ -4943,6 +4979,7 @@ dependencies = [ "ipcc-key-value", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "once_cell", "openapi-lint", "openapiv3", @@ -5020,6 +5057,7 @@ dependencies = [ "omicron-rpaths", "omicron-sled-agent", "omicron-test-utils", + "omicron-workspace-hack", "once_cell", "openapi-lint", "openapiv3", @@ -5099,6 +5137,7 @@ dependencies = [ "omicron-nexus", "omicron-rpaths", "omicron-test-utils", + "omicron-workspace-hack", "pq-sys", "regex", "serde", @@ -5125,6 +5164,7 @@ dependencies = [ "illumos-utils", "indicatif", "omicron-common 0.1.0", + "omicron-workspace-hack", "omicron-zone-package", "petgraph", "rayon", @@ -5154,6 +5194,7 @@ version = "0.1.0" dependencies = [ "argon2", "criterion", + "omicron-workspace-hack", "rand 0.8.5", "rust-argon2", "schemars", @@ -5178,6 +5219,9 @@ dependencies = [ [[package]] name = "omicron-rpaths" version = "0.1.0" +dependencies = [ + "omicron-workspace-hack", +] [[package]] name = "omicron-sled-agent" @@ -5223,6 +5267,7 @@ dependencies = [ "nexus-client 0.1.0", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "once_cell", "openapi-lint", "openapiv3", @@ -5278,6 +5323,7 @@ dependencies = [ "http", "libc", "omicron-common 0.1.0", + "omicron-workspace-hack", "pem", "rcgen", "regex", @@ -5292,6 +5338,110 @@ dependencies = [ "usdt", ] +[[package]] +name = "omicron-workspace-hack" +version = "0.1.0" +dependencies = [ + "anyhow", + "bit-set", + "bit-vec", + "bitflags 1.3.2", + "bitflags 2.4.0", + "bitvec", + "bstr 0.2.17", + "bstr 1.6.0", + "bytes", + "cc", + "chrono", + "cipher", + "clap 4.4.3", + "clap_builder", + "console", + "const-oid", + "crossbeam-epoch", + "crossbeam-utils", + "crypto-common", + "diesel", + "digest", + "either", + "flate2", + "futures", + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", + "gateway-messages", + "generic-array", + "getrandom 0.2.10", + "hashbrown 0.13.2", + "hashbrown 0.14.0", + "hex", + "hyper", + "hyper-rustls", + "indexmap 2.0.0", + "inout", + "ipnetwork", + "itertools 0.10.5", + "lalrpop-util", + "lazy_static", + "libc", + "log", + "managed", + "memchr", + "mio", + "num-bigint", + "num-integer", + "num-iter", + "num-traits", + "once_cell", + "openapiv3", + "petgraph", + "postgres-types", + "ppv-lite86", + "predicates 3.0.3", + "rand 0.8.5", + "rand_chacha 0.3.1", + "regex", + "regex-automata 0.3.8", + "regex-syntax 0.7.5", + "reqwest", + "ring", + "rustix 0.38.9", + "schemars", + "semver 1.0.18", + "serde", + "sha2", + "signature 2.1.0", + "similar", + "slog", + "spin 0.9.8", + "string_cache", + "subtle", + "syn 1.0.109", + "syn 2.0.32", + "textwrap 0.16.0", + "time", + "time-macros", + "tokio", + "tokio-postgres", + "tokio-stream", + "toml 0.7.8", + "toml_datetime", + "toml_edit 0.19.15", + "tracing", + "trust-dns-proto", + "unicode-bidi", + "unicode-normalization", + "unicode-xid", + "usdt", + "uuid", + "yasna", + "zeroize", + "zip", +] + [[package]] name = "omicron-zone-package" version = "0.8.3" @@ -5498,6 +5648,7 @@ dependencies = [ "futures", "http", "hyper", + "omicron-workspace-hack", "progenitor", "rand 0.8.5", "regress", @@ -5532,6 +5683,7 @@ dependencies = [ "chrono", "num", "omicron-common 0.1.0", + "omicron-workspace-hack", "oximeter-macro-impl 0.1.0", "rstest", "schemars", @@ -5563,6 +5715,7 @@ version = "0.1.0" dependencies = [ "chrono", "omicron-common 0.1.0", + "omicron-workspace-hack", "progenitor", "reqwest", "serde", @@ -5582,6 +5735,7 @@ dependencies = [ "nexus-client 0.1.0", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "openapi-lint", "openapiv3", "oximeter 0.1.0", @@ -5610,6 +5764,7 @@ dependencies = [ "dropshot", "itertools 0.11.0", "omicron-test-utils", + "omicron-workspace-hack", "oximeter 0.1.0", "regex", "reqwest", @@ -5634,6 +5789,7 @@ dependencies = [ "dropshot", "futures", "http", + "omicron-workspace-hack", "oximeter 0.1.0", "tokio", "uuid", @@ -5643,6 +5799,7 @@ dependencies = [ name = "oximeter-macro-impl" version = "0.1.0" dependencies = [ + "omicron-workspace-hack", "proc-macro2", "quote", "syn 2.0.32", @@ -5666,6 +5823,7 @@ dependencies = [ "dropshot", "nexus-client 0.1.0", "omicron-common 0.1.0", + "omicron-workspace-hack", "oximeter 0.1.0", "reqwest", "schemars", @@ -7920,6 +8078,7 @@ dependencies = [ "chrono", "ipnetwork", "omicron-common 0.1.0", + "omicron-workspace-hack", "progenitor", "regress", "reqwest", @@ -7945,6 +8104,7 @@ dependencies = [ "nexus-client 0.1.0", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "rand 0.8.5", "schemars", "serde", @@ -8171,6 +8331,7 @@ dependencies = [ "hex", "omicron-common 0.1.0", "omicron-gateway", + "omicron-workspace-hack", "serde", "slog", "slog-dtrace", @@ -9168,6 +9329,7 @@ dependencies = [ "humantime", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "predicates 3.0.3", "slog", "slog-async", @@ -9196,6 +9358,7 @@ dependencies = [ "itertools 0.11.0", "omicron-common 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "rand 0.8.5", "ring", "serde", @@ -9422,6 +9585,7 @@ dependencies = [ "indicatif", "linear-map", "omicron-test-utils", + "omicron-workspace-hack", "owo-colors", "petgraph", "schemars", @@ -9799,6 +9963,7 @@ dependencies = [ "itertools 0.11.0", "omicron-common 0.1.0", "omicron-passwords 0.1.0", + "omicron-workspace-hack", "once_cell", "owo-colors", "proptest", @@ -9834,6 +9999,7 @@ dependencies = [ "anyhow", "gateway-client", "omicron-common 0.1.0", + "omicron-workspace-hack", "schemars", "serde", "serde_json", @@ -9851,6 +10017,7 @@ dependencies = [ "ciborium", "clap 4.4.3", "crossterm 0.27.0", + "omicron-workspace-hack", "ratatui", "reedline", "serde", @@ -9900,6 +10067,7 @@ dependencies = [ "omicron-common 0.1.0", "omicron-passwords 0.1.0", "omicron-test-utils", + "omicron-workspace-hack", "openapi-lint", "openapiv3", "rand 0.8.5", @@ -9936,6 +10104,7 @@ dependencies = [ "chrono", "installinator-common", "ipnetwork", + "omicron-workspace-hack", "progenitor", "regress", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 9498157b28..d660397d9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ members = [ "wicket", "wicketd-client", "wicketd", + "workspace-hack", ] default-members = [ diff --git a/README.adoc b/README.adoc index 988c1276fd..93d1fa4fb8 100644 --- a/README.adoc +++ b/README.adoc @@ -152,30 +152,17 @@ Many of these components themselves are made up of other packages (e.g., `nexus- Use Cargo's `-p PACKAGE` to check/build/test only the package you're working on. Since people are usually only working on one or two components at a time, you can usually iterate faster this way. -=== Why is Cargo rebuilding stuff all the time? +=== Workspace management -People are often surprised to find Cargo rebuilding stuff that it seems like it's just built, even when the relevant source files haven't changed. +Omicron uses `cargo-hakari` to ensure that all workspace dependencies enable the same set of features. This dramatically improves compilation time when switching between different subsets of packages (e.g. `-p wicket` or `-p nexus-db-model`), because the sets of enabled features remain consistent. -* Say you're iterating on code, running `cargo build -p nexus-db-model` to build just that package. Great, it works. Let's run tests: `cargo nextest run -p nexus-db-model`. Now it's rebuilding some _dependency_ of `nexus-db-model` again?! -* Say you've just run `cargo nextest run -p nexus-db-model`. Now you go run `cargo nextest run -p omicron-nexus`, which uses `nexus-db-model`. You see Cargo building `nexus-db-model` again?! +`cargo hakari` status is checked in CI; if the CI check fails, then update the configuration locally with -This usually has to do with the way Cargo selects package https://doc.rust-lang.org/cargo/reference/features.html[features]. These are essentially tags that are used at build time to include specific code or dependencies. For example, the https://docs.rs/serde/latest/serde/[serde] crate defines a feature called https://docs.rs/crate/serde/latest/features["derive"] that controls whether the `Serialize`/`Deserialize` derive macros will be included. Let's look at how this affects builds. - -TIP: You can use `cargo tree` to inspect a package's dependencies, including features. This is useful for debugging feature-related build issues. - -==== Feature selection differs when building tests - -When you run `cargo build -p nexus-db-model`, Cargo looks at all the packages in the depencency tree of `nexus-db-model` and figures out what features it needs for each one. Let's take the `uuid` package as an example. Cargo takes https://doc.rust-lang.org/cargo/reference/features.html#feature-unification[union of the features required by any of the packages that depend on `uuid` in the whole dependency tree of `nexus-db-model`]. Let's say that's just the "v4" feature. Simple enough. - -When you then run `cargo nextest run -p nexus-db-model`, it does the same thing. Only this time, it's looking at the `dev-dependencies` tree. `nexus-db-model` 's dev-dependencies might include some other package that depends on `uuid` and requires the "v5" feature. Now, Cargo has to rebuild `uuid` -- and anything else that depends on it. - -This is why when using Cargo's check/build/clippy commands, we suggest using `--all-targets`. When you use `cargo build --all-targets`, it builds the tests as well. It's usually not much more time and avoids extra rebuilds when switching back and forth between the default targets and the targets with tests included. - -==== Feature selection differs when building different packages - -People run into a similar problem when switching packages within Omicron. Once you've got `cargo nextest run -p nexus-db-model` working, you may run `cargo nextest run -p omicron-nexus`, which uses `nexus-db-model`. And you may be surprised to see Cargo rebuilding some common dependency like `uuid`. It's the same as above: we're building a different package now. It has a different (larger) dependency tree. That may result in some crate deep in the dependency tree needing some new feature, causing it and all of its dependents to be rebuilt. - -NOTE: https://github.com/rust-lang/cargo/issues/4463[There is interest in changing the way feature selection works in workspaces like Omicron for exactly this reason.] It's been suggested to have an option for Cargo to always look at the features required for all packages in the workspace, rather than just the one you've selected. This could eliminate this particular problem. In the meantime, we mitigate this with heavy use of https://doc.rust-lang.org/cargo/reference/workspaces.html#the-dependencies-table[workspace dependencies], which helps make sure that different packages _within_ Omicron depend on the same set of features for a given dependency. +``` +cargo install cargo-hakari --locked # only needed on the first run +cargo hakari generate +cargo hakari manage-deps +``` === Why am I getting compile errors after I thought I'd already built everything? diff --git a/api_identity/Cargo.toml b/api_identity/Cargo.toml index 761e5e3017..9faf2a1878 100644 --- a/api_identity/Cargo.toml +++ b/api_identity/Cargo.toml @@ -14,3 +14,4 @@ proc-macro = true proc-macro2.workspace = true quote.workspace = true syn.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/bootstore/Cargo.toml b/bootstore/Cargo.toml index d916bf80c7..eefe05c8d6 100644 --- a/bootstore/Cargo.toml +++ b/bootstore/Cargo.toml @@ -36,6 +36,7 @@ zeroize.workspace = true # utils`. Unfortunately, it doesn't appear possible to put the `pq-sys` dep # only in `[dev-dependencies]`. pq-sys = "*" +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] assert_matches.workspace = true diff --git a/bootstrap-agent-client/Cargo.toml b/bootstrap-agent-client/Cargo.toml index f7d3ad2db6..17989a5c5f 100644 --- a/bootstrap-agent-client/Cargo.toml +++ b/bootstrap-agent-client/Cargo.toml @@ -17,3 +17,4 @@ serde.workspace = true sled-hardware.workspace = true slog.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/caboose-util/Cargo.toml b/caboose-util/Cargo.toml index 195bbfd5d7..253d54643d 100644 --- a/caboose-util/Cargo.toml +++ b/caboose-util/Cargo.toml @@ -7,3 +7,4 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true hubtools.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/certificates/Cargo.toml b/certificates/Cargo.toml index 29c4a8bc2e..d20d257e4c 100644 --- a/certificates/Cargo.toml +++ b/certificates/Cargo.toml @@ -12,6 +12,7 @@ openssl-sys.workspace = true thiserror.workspace = true omicron-common.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] omicron-test-utils.workspace = true diff --git a/common/Cargo.toml b/common/Cargo.toml index 492b6ca860..bda88d0d43 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -40,6 +40,7 @@ toml.workspace = true uuid.workspace = true parse-display.workspace = true progenitor.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] camino-tempfile.workspace = true diff --git a/crdb-seed/Cargo.toml b/crdb-seed/Cargo.toml index 01af7cb1d7..fa71fe7e8a 100644 --- a/crdb-seed/Cargo.toml +++ b/crdb-seed/Cargo.toml @@ -13,3 +13,4 @@ omicron-test-utils.workspace = true ring.workspace = true slog.workspace = true tokio.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/ddm-admin-client/Cargo.toml b/ddm-admin-client/Cargo.toml index 6e9ee930a0..3814446b3e 100644 --- a/ddm-admin-client/Cargo.toml +++ b/ddm-admin-client/Cargo.toml @@ -15,6 +15,7 @@ tokio.workspace = true omicron-common.workspace = true sled-hardware.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [build-dependencies] anyhow.workspace = true diff --git a/deploy/Cargo.toml b/deploy/Cargo.toml index 1b8e6a92d8..17bacd6354 100644 --- a/deploy/Cargo.toml +++ b/deploy/Cargo.toml @@ -14,6 +14,7 @@ serde.workspace = true serde_derive.workspace = true thiserror.workspace = true toml.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [[bin]] name = "thing-flinger" diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml index c9ebbe35ad..5b2adde1b2 100644 --- a/dev-tools/omdb/Cargo.toml +++ b/dev-tools/omdb/Cargo.toml @@ -32,6 +32,7 @@ tabled.workspace = true textwrap.workspace = true tokio = { workspace = true, features = [ "full" ] } uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } [dev-dependencies] expectorate.workspace = true diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml index 2061489cbb..95da4d42ef 100644 --- a/dev-tools/omicron-dev/Cargo.toml +++ b/dev-tools/omicron-dev/Cargo.toml @@ -28,6 +28,7 @@ signal-hook-tokio.workspace = true tokio = { workspace = true, features = [ "full" ] } tokio-postgres.workspace = true toml.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } [dev-dependencies] camino-tempfile.workspace = true diff --git a/dev-tools/xtask/src/main.rs b/dev-tools/xtask/src/main.rs index a64c87a570..3e52d742f5 100644 --- a/dev-tools/xtask/src/main.rs +++ b/dev-tools/xtask/src/main.rs @@ -78,6 +78,8 @@ fn cmd_clippy() -> Result<()> { Ok(()) } +const WORKSPACE_HACK_PACKAGE_NAME: &str = "omicron-workspace-hack"; + fn cmd_check_workspace_deps() -> Result<()> { // Ignore issues with "pq-sys". See the omicron-rpaths package for details. const EXCLUDED: &[&'static str] = &["pq-sys"]; @@ -97,6 +99,12 @@ fn cmd_check_workspace_deps() -> Result<()> { // Iterate the workspace packages and fill out the maps above. for pkg_info in workspace.workspace_packages() { + if pkg_info.name == WORKSPACE_HACK_PACKAGE_NAME { + // Skip over workspace-hack because hakari doesn't yet support + // workspace deps: https://github.com/guppy-rs/guppy/issues/7 + continue; + } + let manifest_path = &pkg_info.manifest_path; let manifest = read_cargo_toml(manifest_path)?; for tree in [ @@ -125,6 +133,12 @@ fn cmd_check_workspace_deps() -> Result<()> { } } + if name == WORKSPACE_HACK_PACKAGE_NAME { + // Skip over workspace-hack because hakari doesn't yet support + // workspace deps: https://github.com/guppy-rs/guppy/issues/7 + continue; + } + non_workspace_dependencies .entry(name.to_owned()) .or_insert_with(Vec::new) diff --git a/dns-server/Cargo.toml b/dns-server/Cargo.toml index 243876a5a2..d7606dcff5 100644 --- a/dns-server/Cargo.toml +++ b/dns-server/Cargo.toml @@ -30,6 +30,7 @@ trust-dns-proto.workspace = true trust-dns-resolver.workspace = true trust-dns-server.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] expectorate.workspace = true diff --git a/dns-service-client/Cargo.toml b/dns-service-client/Cargo.toml index 7f5cf63d6a..e351d90da2 100644 --- a/dns-service-client/Cargo.toml +++ b/dns-service-client/Cargo.toml @@ -14,3 +14,4 @@ serde.workspace = true serde_json.workspace = true slog.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/dpd-client/Cargo.toml b/dpd-client/Cargo.toml index fdbdcd07af..26807f7d79 100644 --- a/dpd-client/Cargo.toml +++ b/dpd-client/Cargo.toml @@ -17,6 +17,7 @@ ipnetwork.workspace = true http.workspace = true schemars.workspace = true rand.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [build-dependencies] anyhow.workspace = true diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml index a0e099b756..5ff0f9b377 100644 --- a/end-to-end-tests/Cargo.toml +++ b/end-to-end-tests/Cargo.toml @@ -24,3 +24,4 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } toml.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/gateway-cli/Cargo.toml b/gateway-cli/Cargo.toml index d5083d1999..0d179750ea 100644 --- a/gateway-cli/Cargo.toml +++ b/gateway-cli/Cargo.toml @@ -24,3 +24,4 @@ uuid.workspace = true gateway-client.workspace = true gateway-messages.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/gateway-client/Cargo.toml b/gateway-client/Cargo.toml index 81d1630a1d..96a1eb221f 100644 --- a/gateway-client/Cargo.toml +++ b/gateway-client/Cargo.toml @@ -15,3 +15,4 @@ serde_json.workspace = true schemars.workspace = true slog.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/gateway-test-utils/Cargo.toml b/gateway-test-utils/Cargo.toml index 8f6e14d68a..9d80e63f05 100644 --- a/gateway-test-utils/Cargo.toml +++ b/gateway-test-utils/Cargo.toml @@ -14,3 +14,4 @@ slog.workspace = true sp-sim.workspace = true tokio.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 307baa3f27..f5abce88e9 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -34,6 +34,7 @@ tokio-tungstenite.workspace = true tokio-util.workspace = true toml.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] expectorate.workspace = true diff --git a/illumos-utils/Cargo.toml b/illumos-utils/Cargo.toml index 3c0c2e7fc9..e292097bc5 100644 --- a/illumos-utils/Cargo.toml +++ b/illumos-utils/Cargo.toml @@ -29,6 +29,7 @@ zone.workspace = true # only enabled via the `testing` feature mockall = { workspace = true, optional = true } +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(target_os = "illumos")'.dependencies] opte-ioctl.workspace = true diff --git a/installinator-artifact-client/Cargo.toml b/installinator-artifact-client/Cargo.toml index ddbc106ee8..18447b8e83 100644 --- a/installinator-artifact-client/Cargo.toml +++ b/installinator-artifact-client/Cargo.toml @@ -15,3 +15,4 @@ serde_json.workspace = true slog.workspace = true update-engine.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/installinator-artifactd/Cargo.toml b/installinator-artifactd/Cargo.toml index 3ce6112165..9318b725db 100644 --- a/installinator-artifactd/Cargo.toml +++ b/installinator-artifactd/Cargo.toml @@ -20,6 +20,7 @@ uuid.workspace = true installinator-common.workspace = true omicron-common.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] expectorate.workspace = true diff --git a/installinator-common/Cargo.toml b/installinator-common/Cargo.toml index ff664f28a3..0f1bf86901 100644 --- a/installinator-common/Cargo.toml +++ b/installinator-common/Cargo.toml @@ -15,3 +15,4 @@ serde_json.workspace = true serde_with.workspace = true thiserror.workspace = true update-engine.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml index c0e7625e6e..3b2f04c38f 100644 --- a/installinator/Cargo.toml +++ b/installinator/Cargo.toml @@ -42,6 +42,7 @@ toml.workspace = true tufaceous-lib.workspace = true update-engine.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] omicron-test-utils.workspace = true diff --git a/internal-dns-cli/Cargo.toml b/internal-dns-cli/Cargo.toml index d922544722..fb5780d22a 100644 --- a/internal-dns-cli/Cargo.toml +++ b/internal-dns-cli/Cargo.toml @@ -13,3 +13,4 @@ omicron-common.workspace = true slog.workspace = true tokio.workspace = true trust-dns-resolver.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/internal-dns/Cargo.toml b/internal-dns/Cargo.toml index 5ead1cc8a4..d680ab3ce1 100644 --- a/internal-dns/Cargo.toml +++ b/internal-dns/Cargo.toml @@ -17,6 +17,7 @@ thiserror.workspace = true trust-dns-proto.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] assert_matches.workspace = true diff --git a/ipcc-key-value/Cargo.toml b/ipcc-key-value/Cargo.toml index a3f17cea52..128fde9a01 100644 --- a/ipcc-key-value/Cargo.toml +++ b/ipcc-key-value/Cargo.toml @@ -11,6 +11,7 @@ omicron-common.workspace = true serde.workspace = true thiserror.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] omicron-common = { workspace = true, features = ["testing"] } diff --git a/key-manager/Cargo.toml b/key-manager/Cargo.toml index 7954a977a3..69ae3b25bd 100644 --- a/key-manager/Cargo.toml +++ b/key-manager/Cargo.toml @@ -14,4 +14,5 @@ slog.workspace = true thiserror.workspace = true tokio.workspace = true zeroize.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/nexus-client/Cargo.toml b/nexus-client/Cargo.toml index 589562c930..d59c013992 100644 --- a/nexus-client/Cargo.toml +++ b/nexus-client/Cargo.toml @@ -18,3 +18,4 @@ serde.workspace = true serde_json.workspace = true slog.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 1a09f07f6c..91872e2c32 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -90,6 +90,7 @@ oximeter.workspace = true oximeter-instruments = { workspace = true, features = ["http-instruments"] } oximeter-producer.workspace = true rustls = { workspace = true } +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] async-bb8-diesel.workspace = true diff --git a/nexus/authz-macros/Cargo.toml b/nexus/authz-macros/Cargo.toml index 40303b2e34..3d55afa477 100644 --- a/nexus/authz-macros/Cargo.toml +++ b/nexus/authz-macros/Cargo.toml @@ -14,3 +14,4 @@ quote.workspace = true serde.workspace = true serde_tokenstream.workspace = true syn.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } diff --git a/nexus/db-macros/Cargo.toml b/nexus/db-macros/Cargo.toml index 3fb228f26c..ce206bb56e 100644 --- a/nexus/db-macros/Cargo.toml +++ b/nexus/db-macros/Cargo.toml @@ -15,6 +15,7 @@ quote.workspace = true serde.workspace = true serde_tokenstream.workspace = true syn = { workspace = true, features = ["extra-traits"] } +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } [dev-dependencies] rustfmt-wrapper.workspace = true diff --git a/nexus/db-model/Cargo.toml b/nexus/db-model/Cargo.toml index dc83670725..aedbb9168b 100644 --- a/nexus/db-model/Cargo.toml +++ b/nexus/db-model/Cargo.toml @@ -36,6 +36,7 @@ nexus-defaults.workspace = true nexus-types.workspace = true omicron-passwords.workspace = true sled-agent-client.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } [dev-dependencies] expectorate.workspace = true diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index a8256cb60a..af01c1732b 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -63,6 +63,7 @@ nexus-types.workspace = true omicron-common.workspace = true omicron-passwords.workspace = true oximeter.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } [dev-dependencies] assert_matches.workspace = true diff --git a/nexus/defaults/Cargo.toml b/nexus/defaults/Cargo.toml index 910ae2afd6..09a95fa839 100644 --- a/nexus/defaults/Cargo.toml +++ b/nexus/defaults/Cargo.toml @@ -11,3 +11,4 @@ rand.workspace = true serde_json.workspace = true omicron-common.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } diff --git a/nexus/test-interface/Cargo.toml b/nexus/test-interface/Cargo.toml index 44c894411b..e0743e84bc 100644 --- a/nexus/test-interface/Cargo.toml +++ b/nexus/test-interface/Cargo.toml @@ -12,3 +12,4 @@ nexus-types.workspace = true omicron-common.workspace = true slog.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } diff --git a/nexus/test-utils-macros/Cargo.toml b/nexus/test-utils-macros/Cargo.toml index 4f9d3eca32..1bfa25017a 100644 --- a/nexus/test-utils-macros/Cargo.toml +++ b/nexus/test-utils-macros/Cargo.toml @@ -11,3 +11,4 @@ proc-macro = true proc-macro2.workspace = true quote.workspace = true syn = { workspace = true, features = [ "fold", "parsing" ] } +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index bad225516a..a2e7600e93 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -38,3 +38,4 @@ tempfile.workspace = true trust-dns-proto.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index c0f175cf31..f7ffafec52 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -25,3 +25,4 @@ api_identity.workspace = true dns-service-client.workspace = true omicron-common.workspace = true omicron-passwords.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } diff --git a/oxide-client/Cargo.toml b/oxide-client/Cargo.toml index 0602066e6d..df34ab9721 100644 --- a/oxide-client/Cargo.toml +++ b/oxide-client/Cargo.toml @@ -21,3 +21,4 @@ thiserror.workspace = true tokio = { workspace = true, features = [ "net" ] } trust-dns-resolver.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/oximeter-client/Cargo.toml b/oximeter-client/Cargo.toml index e4e68464d7..297dfb6c92 100644 --- a/oximeter-client/Cargo.toml +++ b/oximeter-client/Cargo.toml @@ -12,3 +12,4 @@ reqwest = { workspace = true, features = ["json", "rustls-tls", "stream"] } serde.workspace = true slog.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/oximeter/collector/Cargo.toml b/oximeter/collector/Cargo.toml index 1137651aa0..c8c4030dba 100644 --- a/oximeter/collector/Cargo.toml +++ b/oximeter/collector/Cargo.toml @@ -22,6 +22,7 @@ thiserror.workspace = true tokio.workspace = true toml.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } [dev-dependencies] expectorate.workspace = true diff --git a/oximeter/db/Cargo.toml b/oximeter/db/Cargo.toml index 9ff4ac5c06..77bce09db9 100644 --- a/oximeter/db/Cargo.toml +++ b/oximeter/db/Cargo.toml @@ -25,6 +25,7 @@ thiserror.workspace = true tokio = { workspace = true, features = [ "rt-multi-thread", "macros" ] } usdt.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } [dev-dependencies] itertools.workspace = true diff --git a/oximeter/instruments/Cargo.toml b/oximeter/instruments/Cargo.toml index 98f8f3b5b2..4adff0463a 100644 --- a/oximeter/instruments/Cargo.toml +++ b/oximeter/instruments/Cargo.toml @@ -12,6 +12,7 @@ oximeter.workspace = true tokio.workspace = true http = { workspace = true, optional = true } uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } [features] default = ["http-instruments"] diff --git a/oximeter/oximeter-macro-impl/Cargo.toml b/oximeter/oximeter-macro-impl/Cargo.toml index c38d85ed2d..ff116e1c9d 100644 --- a/oximeter/oximeter-macro-impl/Cargo.toml +++ b/oximeter/oximeter-macro-impl/Cargo.toml @@ -12,3 +12,4 @@ proc-macro = true proc-macro2.workspace = true quote.workspace = true syn = { workspace = true, features = [ "full", "extra-traits" ] } +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } diff --git a/oximeter/oximeter/Cargo.toml b/oximeter/oximeter/Cargo.toml index f0549548a6..b2aa15f85e 100644 --- a/oximeter/oximeter/Cargo.toml +++ b/oximeter/oximeter/Cargo.toml @@ -15,6 +15,7 @@ schemars = { workspace = true, features = [ "uuid1", "bytes", "chrono" ] } serde.workspace = true thiserror.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } [dev-dependencies] approx.workspace = true diff --git a/oximeter/producer/Cargo.toml b/oximeter/producer/Cargo.toml index e511294e52..f171f57e8a 100644 --- a/oximeter/producer/Cargo.toml +++ b/oximeter/producer/Cargo.toml @@ -19,3 +19,4 @@ slog-dtrace.workspace = true tokio.workspace = true thiserror.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } diff --git a/package/Cargo.toml b/package/Cargo.toml index 7c786b77ef..9fc4610020 100644 --- a/package/Cargo.toml +++ b/package/Cargo.toml @@ -34,6 +34,7 @@ tokio = { workspace = true, features = [ "full" ] } toml.workspace = true topological-sort.workspace = true walkdir.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] expectorate.workspace = true diff --git a/passwords/Cargo.toml b/passwords/Cargo.toml index 1731716101..cbd569ef4c 100644 --- a/passwords/Cargo.toml +++ b/passwords/Cargo.toml @@ -11,6 +11,7 @@ thiserror.workspace = true schemars.workspace = true serde.workspace = true serde_with.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] argon2alt = { package = "rust-argon2", version = "1.0" } diff --git a/rpaths/Cargo.toml b/rpaths/Cargo.toml index 829b4ffe28..7671be4968 100644 --- a/rpaths/Cargo.toml +++ b/rpaths/Cargo.toml @@ -5,3 +5,4 @@ edition = "2021" license = "MPL-2.0" [dependencies] +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/sled-agent-client/Cargo.toml b/sled-agent-client/Cargo.toml index f6c58fbf2b..01c1032a51 100644 --- a/sled-agent-client/Cargo.toml +++ b/sled-agent-client/Cargo.toml @@ -15,3 +15,4 @@ reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } serde.workspace = true slog.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index f172136726..b131698395 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -76,6 +76,7 @@ uuid.workspace = true zeroize.workspace = true zone.workspace = true static_assertions.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(target_os = "illumos")'.dependencies] opte-ioctl.workspace = true diff --git a/sled-hardware/Cargo.toml b/sled-hardware/Cargo.toml index c6bc09f41e..880f93441c 100644 --- a/sled-hardware/Cargo.toml +++ b/sled-hardware/Cargo.toml @@ -24,6 +24,7 @@ thiserror.workspace = true tofino.workspace = true tokio.workspace = true uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(target_os = "illumos")'.dependencies] illumos-devinfo = { git = "https://github.com/oxidecomputer/illumos-devinfo", branch = "main" } diff --git a/sp-sim/Cargo.toml b/sp-sim/Cargo.toml index 5a73f46d9e..2a1ae19468 100644 --- a/sp-sim/Cargo.toml +++ b/sp-sim/Cargo.toml @@ -21,6 +21,7 @@ sprockets-rot.workspace = true thiserror.workspace = true tokio = { workspace = true, features = [ "full" ] } toml.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [[bin]] name = "sp-sim" diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 09ff12a806..a0227a4de2 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -25,6 +25,7 @@ usdt.workspace = true rcgen.workspace = true regex.workspace = true reqwest.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] expectorate.workspace = true diff --git a/tufaceous-lib/Cargo.toml b/tufaceous-lib/Cargo.toml index e4799a69a4..8b5c4fa7ca 100644 --- a/tufaceous-lib/Cargo.toml +++ b/tufaceous-lib/Cargo.toml @@ -32,6 +32,7 @@ toml.workspace = true tough.workspace = true url = "2.4.1" zip.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] omicron-test-utils.workspace = true diff --git a/tufaceous/Cargo.toml b/tufaceous/Cargo.toml index 09772daef4..f3e3b815d2 100644 --- a/tufaceous/Cargo.toml +++ b/tufaceous/Cargo.toml @@ -18,6 +18,7 @@ slog-async.workspace = true slog-envlogger.workspace = true slog-term.workspace = true tufaceous-lib.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] assert_cmd.workspace = true diff --git a/update-engine/Cargo.toml b/update-engine/Cargo.toml index 4c2841cf0f..25ade83f34 100644 --- a/update-engine/Cargo.toml +++ b/update-engine/Cargo.toml @@ -21,6 +21,7 @@ schemars = { workspace = true, features = ["uuid1"] } slog.workspace = true tokio = { workspace = true, features = ["macros", "sync", "time", "rt-multi-thread"] } uuid.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] buf-list.workspace = true diff --git a/wicket-common/Cargo.toml b/wicket-common/Cargo.toml index 735f4a758e..229561cd38 100644 --- a/wicket-common/Cargo.toml +++ b/wicket-common/Cargo.toml @@ -13,3 +13,4 @@ serde.workspace = true serde_json.workspace = true thiserror.workspace = true update-engine.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/wicket-dbg/Cargo.toml b/wicket-dbg/Cargo.toml index 1aa8e10171..bc22424c69 100644 --- a/wicket-dbg/Cargo.toml +++ b/wicket-dbg/Cargo.toml @@ -22,6 +22,7 @@ wicket.workspace = true # used only by wicket-dbg binary reedline = "0.23.0" +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [[bin]] name = "wicket-dbg" diff --git a/wicket/Cargo.toml b/wicket/Cargo.toml index d2004e0a68..58605c8037 100644 --- a/wicket/Cargo.toml +++ b/wicket/Cargo.toml @@ -46,6 +46,7 @@ omicron-passwords.workspace = true update-engine.workspace = true wicket-common.workspace = true wicketd-client.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [dev-dependencies] assert_cmd.workspace = true diff --git a/wicketd-client/Cargo.toml b/wicketd-client/Cargo.toml index 69a7f8fae4..2d959f1f8d 100644 --- a/wicketd-client/Cargo.toml +++ b/wicketd-client/Cargo.toml @@ -18,3 +18,4 @@ slog.workspace = true update-engine.workspace = true uuid.workspace = true wicket-common.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index 8f4faf6c40..a36344b6fb 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -53,6 +53,7 @@ sled-hardware.workspace = true tufaceous-lib.workspace = true update-engine.workspace = true wicket-common.workspace = true +omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } [[bin]] name = "wicketd" diff --git a/workspace-hack/.gitattributes b/workspace-hack/.gitattributes new file mode 100644 index 0000000000..3e9dba4b64 --- /dev/null +++ b/workspace-hack/.gitattributes @@ -0,0 +1,4 @@ +# Avoid putting conflict markers in the generated Cargo.toml file, since their presence breaks +# Cargo. +# Also do not check out the file as CRLF on Windows, as that's what hakari needs. +Cargo.toml merge=binary -crlf diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml new file mode 100644 index 0000000000..d3e00b1831 --- /dev/null +++ b/workspace-hack/Cargo.toml @@ -0,0 +1,261 @@ +# This file is generated by `cargo hakari`. +# To regenerate, run: +# cargo hakari generate + +[package] +name = "omicron-workspace-hack" +version = "0.1.0" +description = "workspace-hack package, managed by hakari" +# You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing. +publish = false + +# The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments +# are managed by hakari. + +### BEGIN HAKARI SECTION +[dependencies] +anyhow = { version = "1", features = ["backtrace"] } +bit-set = { version = "0.5" } +bit-vec = { version = "0.6" } +bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1" } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["serde"] } +bitvec = { version = "1" } +bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2" } +bstr-dff4ba8e3ae991db = { package = "bstr", version = "1" } +bytes = { version = "1", features = ["serde"] } +chrono = { version = "0.4", features = ["alloc", "serde"] } +cipher = { version = "0.4", default-features = false, features = ["block-padding", "zeroize"] } +clap = { version = "4", features = ["derive", "env", "wrap_help"] } +clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "suggestions", "usage", "wrap_help"] } +console = { version = "0.15" } +const-oid = { version = "0.9", default-features = false, features = ["db", "std"] } +crossbeam-epoch = { version = "0.9" } +crossbeam-utils = { version = "0.8" } +crypto-common = { version = "0.1", default-features = false, features = ["getrandom", "std"] } +diesel = { version = "2", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } +digest = { version = "0.10", features = ["mac", "oid", "std"] } +either = { version = "1" } +flate2 = { version = "1" } +futures = { version = "0.3" } +futures-channel = { version = "0.3", features = ["sink"] } +futures-core = { version = "0.3" } +futures-io = { version = "0.3", default-features = false, features = ["std"] } +futures-sink = { version = "0.3" } +futures-task = { version = "0.3", default-features = false, features = ["std"] } +futures-util = { version = "0.3", features = ["channel", "io", "sink"] } +gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", features = ["std"] } +generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] } +getrandom = { version = "0.2", default-features = false, features = ["js", "rdrand", "std"] } +hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } +hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13" } +hex = { version = "0.4", features = ["serde"] } +hyper = { version = "0.14", features = ["full"] } +indexmap = { version = "2", features = ["serde"] } +inout = { version = "0.1", default-features = false, features = ["std"] } +ipnetwork = { version = "0.20", features = ["schemars"] } +itertools = { version = "0.10" } +lalrpop-util = { version = "0.19" } +lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } +libc = { version = "0.2", features = ["extra_traits"] } +log = { version = "0.4", default-features = false, features = ["std"] } +managed = { version = "0.8", default-features = false, features = ["alloc", "map"] } +memchr = { version = "2" } +num-bigint = { version = "0.4", features = ["rand"] } +num-integer = { version = "0.1", features = ["i128"] } +num-iter = { version = "0.1", default-features = false, features = ["i128"] } +num-traits = { version = "0.2", features = ["i128", "libm"] } +openapiv3 = { version = "1", default-features = false, features = ["skip_serializing_defaults"] } +petgraph = { version = "0.6", features = ["serde-1"] } +postgres-types = { version = "0.2", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } +ppv-lite86 = { version = "0.2", default-features = false, features = ["simd", "std"] } +predicates = { version = "3" } +rand = { version = "0.8", features = ["min_const_gen"] } +rand_chacha = { version = "0.3" } +regex = { version = "1" } +regex-automata = { version = "0.3", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } +regex-syntax = { version = "0.7" } +reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls", "stream"] } +ring = { version = "0.16", features = ["std"] } +schemars = { version = "0.8", features = ["bytes", "chrono", "uuid1"] } +semver = { version = "1", features = ["serde"] } +serde = { version = "1", features = ["alloc", "derive", "rc"] } +sha2 = { version = "0.10", features = ["oid"] } +signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] } +similar = { version = "2", features = ["inline", "unicode"] } +slog = { version = "2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } +spin = { version = "0.9" } +string_cache = { version = "0.8" } +subtle = { version = "2" } +syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "fold", "full", "visit"] } +syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +textwrap = { version = "0.16" } +time = { version = "0.3", features = ["formatting", "local-offset", "macros", "parsing"] } +tokio = { version = "1", features = ["full", "test-util"] } +tokio-postgres = { version = "0.7", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } +tokio-stream = { version = "0.1", features = ["net"] } +toml = { version = "0.7" } +toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } +toml_edit = { version = "0.19", features = ["serde"] } +tracing = { version = "0.1", features = ["log"] } +trust-dns-proto = { version = "0.22" } +unicode-bidi = { version = "0.3" } +unicode-normalization = { version = "0.1" } +usdt = { version = "0.3" } +uuid = { version = "1", features = ["serde", "v4"] } +yasna = { version = "0.5", features = ["bit-vec", "num-bigint", "std", "time"] } +zeroize = { version = "1", features = ["std", "zeroize_derive"] } +zip = { version = "0.6", default-features = false, features = ["bzip2", "deflate"] } + +[build-dependencies] +anyhow = { version = "1", features = ["backtrace"] } +bit-set = { version = "0.5" } +bit-vec = { version = "0.6" } +bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1" } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["serde"] } +bitvec = { version = "1" } +bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2" } +bstr-dff4ba8e3ae991db = { package = "bstr", version = "1" } +bytes = { version = "1", features = ["serde"] } +cc = { version = "1", default-features = false, features = ["parallel"] } +chrono = { version = "0.4", features = ["alloc", "serde"] } +cipher = { version = "0.4", default-features = false, features = ["block-padding", "zeroize"] } +clap = { version = "4", features = ["derive", "env", "wrap_help"] } +clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "suggestions", "usage", "wrap_help"] } +console = { version = "0.15" } +const-oid = { version = "0.9", default-features = false, features = ["db", "std"] } +crossbeam-epoch = { version = "0.9" } +crossbeam-utils = { version = "0.8" } +crypto-common = { version = "0.1", default-features = false, features = ["getrandom", "std"] } +diesel = { version = "2", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } +digest = { version = "0.10", features = ["mac", "oid", "std"] } +either = { version = "1" } +flate2 = { version = "1" } +futures = { version = "0.3" } +futures-channel = { version = "0.3", features = ["sink"] } +futures-core = { version = "0.3" } +futures-io = { version = "0.3", default-features = false, features = ["std"] } +futures-sink = { version = "0.3" } +futures-task = { version = "0.3", default-features = false, features = ["std"] } +futures-util = { version = "0.3", features = ["channel", "io", "sink"] } +gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", features = ["std"] } +generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] } +getrandom = { version = "0.2", default-features = false, features = ["js", "rdrand", "std"] } +hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } +hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13" } +hex = { version = "0.4", features = ["serde"] } +hyper = { version = "0.14", features = ["full"] } +indexmap = { version = "2", features = ["serde"] } +inout = { version = "0.1", default-features = false, features = ["std"] } +ipnetwork = { version = "0.20", features = ["schemars"] } +itertools = { version = "0.10" } +lalrpop-util = { version = "0.19" } +lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } +libc = { version = "0.2", features = ["extra_traits"] } +log = { version = "0.4", default-features = false, features = ["std"] } +managed = { version = "0.8", default-features = false, features = ["alloc", "map"] } +memchr = { version = "2" } +num-bigint = { version = "0.4", features = ["rand"] } +num-integer = { version = "0.1", features = ["i128"] } +num-iter = { version = "0.1", default-features = false, features = ["i128"] } +num-traits = { version = "0.2", features = ["i128", "libm"] } +openapiv3 = { version = "1", default-features = false, features = ["skip_serializing_defaults"] } +petgraph = { version = "0.6", features = ["serde-1"] } +postgres-types = { version = "0.2", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } +ppv-lite86 = { version = "0.2", default-features = false, features = ["simd", "std"] } +predicates = { version = "3" } +rand = { version = "0.8", features = ["min_const_gen"] } +rand_chacha = { version = "0.3" } +regex = { version = "1" } +regex-automata = { version = "0.3", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } +regex-syntax = { version = "0.7" } +reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls", "stream"] } +ring = { version = "0.16", features = ["std"] } +schemars = { version = "0.8", features = ["bytes", "chrono", "uuid1"] } +semver = { version = "1", features = ["serde"] } +serde = { version = "1", features = ["alloc", "derive", "rc"] } +sha2 = { version = "0.10", features = ["oid"] } +signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] } +similar = { version = "2", features = ["inline", "unicode"] } +slog = { version = "2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } +spin = { version = "0.9" } +string_cache = { version = "0.8" } +subtle = { version = "2" } +syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "fold", "full", "visit"] } +syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +textwrap = { version = "0.16" } +time = { version = "0.3", features = ["formatting", "local-offset", "macros", "parsing"] } +time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing"] } +tokio = { version = "1", features = ["full", "test-util"] } +tokio-postgres = { version = "0.7", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } +tokio-stream = { version = "0.1", features = ["net"] } +toml = { version = "0.7" } +toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } +toml_edit = { version = "0.19", features = ["serde"] } +tracing = { version = "0.1", features = ["log"] } +trust-dns-proto = { version = "0.22" } +unicode-bidi = { version = "0.3" } +unicode-normalization = { version = "0.1" } +unicode-xid = { version = "0.2" } +usdt = { version = "0.3" } +uuid = { version = "1", features = ["serde", "v4"] } +yasna = { version = "0.5", features = ["bit-vec", "num-bigint", "std", "time"] } +zeroize = { version = "1", features = ["std", "zeroize_derive"] } +zip = { version = "0.6", default-features = false, features = ["bzip2", "deflate"] } + +[target.x86_64-unknown-linux-gnu.dependencies] +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24" } +mio = { version = "0.8", features = ["net", "os-ext"] } +once_cell = { version = "1", features = ["unstable"] } +rustix = { version = "0.38", features = ["fs", "termios"] } + +[target.x86_64-unknown-linux-gnu.build-dependencies] +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24" } +mio = { version = "0.8", features = ["net", "os-ext"] } +once_cell = { version = "1", features = ["unstable"] } +rustix = { version = "0.38", features = ["fs", "termios"] } + +[target.x86_64-apple-darwin.dependencies] +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24" } +mio = { version = "0.8", features = ["net", "os-ext"] } +once_cell = { version = "1", features = ["unstable"] } +rustix = { version = "0.38", features = ["fs", "termios"] } + +[target.x86_64-apple-darwin.build-dependencies] +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24" } +mio = { version = "0.8", features = ["net", "os-ext"] } +once_cell = { version = "1", features = ["unstable"] } +rustix = { version = "0.38", features = ["fs", "termios"] } + +[target.aarch64-apple-darwin.dependencies] +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24" } +mio = { version = "0.8", features = ["net", "os-ext"] } +once_cell = { version = "1", features = ["unstable"] } +rustix = { version = "0.38", features = ["fs", "termios"] } + +[target.aarch64-apple-darwin.build-dependencies] +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24" } +mio = { version = "0.8", features = ["net", "os-ext"] } +once_cell = { version = "1", features = ["unstable"] } +rustix = { version = "0.38", features = ["fs", "termios"] } + +[target.x86_64-unknown-illumos.dependencies] +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24" } +mio = { version = "0.8", features = ["net", "os-ext"] } +once_cell = { version = "1", features = ["unstable"] } +rustix = { version = "0.38", features = ["fs", "termios"] } + +[target.x86_64-unknown-illumos.build-dependencies] +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24" } +mio = { version = "0.8", features = ["net", "os-ext"] } +once_cell = { version = "1", features = ["unstable"] } +rustix = { version = "0.38", features = ["fs", "termios"] } + +### END HAKARI SECTION diff --git a/workspace-hack/build.rs b/workspace-hack/build.rs new file mode 100644 index 0000000000..92518ef04c --- /dev/null +++ b/workspace-hack/build.rs @@ -0,0 +1,2 @@ +// A build script is required for cargo to consider build dependencies. +fn main() {} diff --git a/workspace-hack/src/lib.rs b/workspace-hack/src/lib.rs new file mode 100644 index 0000000000..22489f632b --- /dev/null +++ b/workspace-hack/src/lib.rs @@ -0,0 +1 @@ +// This is a stub lib.rs. From b03dd6b740a2b6545a0de847b74847b3e2199800 Mon Sep 17 00:00:00 2001 From: Rain Date: Thu, 28 Sep 2023 20:13:43 -0700 Subject: [PATCH 04/89] [wicketd] allow starting multiple updates with one API call (#4039) Extend `post_start_update` to allow starting updates on several sleds at once. This is not currently used (the TUI always updates one sled at a time), but will be used for command-line driven mupdates. If we're issuing updates on several sleds at once, we can encounter different kinds of errors for each sled. So instead of returning immediately, we collect errors into a vector and then return them all at once. This also required some refactoring in `update_tracker.rs`. To take care of all possible situations: 1. Add a new `SpawnUpdateDriver` trait, which has two methods: one to perform a one-time setup, and one to perform a spawn operation for each SP. 2. Add three implementations of `SpawnUpdateDriver`: `RealUpdateDriver` which is the actual implementation, `FakeUpdateDriver` which is used for tests, and `NeverUpdateDriver` which is an uninhabited type (empty enum, can never be constructed) and is used to perform pre-update checks but not the update itself. Happy to hear suggestions about how to make this better. One path I went down but rejected is using a typestate to indicate that update checks had passed -- then the caller could decide whether to perform the update or not. The problem is that for the typestate to be valid it would have to hold on to the `MutexGuard` (otherwise something could come in between and replace the task that we thought was finished), and that seems a bit fraught as you could accidentally attempt to lock the update data again. A callback-like approach, which was the previous implementation and which has been retained in this PR, does not have that pitfall. I tested this by spinning up sp-sim, mgs, and wicketd, and it worked as expected. Errors (e.g. no inventory present) were caught as expected. --- Cargo.lock | 1 + openapi/wicketd.json | 97 ++-- wicket/src/wicketd.rs | 11 +- wicketd/Cargo.toml | 1 + wicketd/src/helpers.rs | 41 ++ wicketd/src/http_entrypoints.rs | 219 ++++++--- wicketd/src/lib.rs | 1 + wicketd/src/update_tracker.rs | 499 ++++++++++++++------- wicketd/tests/integration_tests/updates.rs | 27 +- 9 files changed, 595 insertions(+), 302 deletions(-) create mode 100644 wicketd/src/helpers.rs diff --git a/Cargo.lock b/Cargo.lock index 5f073db250..138080640e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10063,6 +10063,7 @@ dependencies = [ "installinator-artifact-client", "installinator-artifactd", "installinator-common", + "itertools 0.11.0", "omicron-certificates", "omicron-common 0.1.0", "omicron-passwords 0.1.0", diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 40d798da00..d67fc79f7a 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -598,6 +598,33 @@ } } }, + "/update": { + "post": { + "summary": "An endpoint to start updating one or more sleds, switches and PSCs.", + "operationId": "post_start_update", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StartUpdateParams" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/update/{type}/{slot}": { "get": { "summary": "An endpoint to get the status of any update being performed or recently", @@ -641,51 +668,6 @@ "$ref": "#/components/responses/Error" } } - }, - "post": { - "summary": "An endpoint to start updating a sled.", - "operationId": "post_start_update", - "parameters": [ - { - "in": "path", - "name": "slot", - "required": true, - "schema": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "in": "path", - "name": "type", - "required": true, - "schema": { - "$ref": "#/components/schemas/SpType" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/StartUpdateOptions" - } - } - }, - "required": true - }, - "responses": { - "204": { - "description": "resource updated" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } } } }, @@ -2761,6 +2743,31 @@ "skip_sp_version_check" ] }, + "StartUpdateParams": { + "type": "object", + "properties": { + "options": { + "description": "Options for the update.", + "allOf": [ + { + "$ref": "#/components/schemas/StartUpdateOptions" + } + ] + }, + "targets": { + "description": "The SP identifiers to start the update with. Must be non-empty.", + "type": "array", + "items": { + "$ref": "#/components/schemas/SpIdentifier" + }, + "uniqueItems": true + } + }, + "required": [ + "options", + "targets" + ] + }, "StepComponentSummaryForGenericSpec": { "type": "object", "properties": { diff --git a/wicket/src/wicketd.rs b/wicket/src/wicketd.rs index 160bcb1c6a..2411542429 100644 --- a/wicket/src/wicketd.rs +++ b/wicket/src/wicketd.rs @@ -12,7 +12,7 @@ use tokio::time::{interval, Duration, MissedTickBehavior}; use wicketd_client::types::{ AbortUpdateOptions, ClearUpdateStateOptions, GetInventoryParams, GetInventoryResponse, GetLocationResponse, IgnitionCommand, SpIdentifier, - SpType, StartUpdateOptions, + SpType, StartUpdateOptions, StartUpdateParams, }; use crate::events::EventReportMap; @@ -164,10 +164,11 @@ impl WicketdManager { tokio::spawn(async move { let update_client = create_wicketd_client(&log, addr, WICKETD_TIMEOUT); - let sp: SpIdentifier = component_id.into(); - let response = match update_client - .post_start_update(sp.type_, sp.slot, &options) - .await + let params = StartUpdateParams { + targets: vec![component_id.into()], + options, + }; + let response = match update_client.post_start_update(¶ms).await { Ok(_) => Ok(()), Err(error) => Err(error.to_string()), diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index a36344b6fb..6df5e0e4e5 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -24,6 +24,7 @@ hubtools.workspace = true http.workspace = true hyper.workspace = true illumos-utils.workspace = true +itertools.workspace = true reqwest.workspace = true schemars.workspace = true serde.workspace = true diff --git a/wicketd/src/helpers.rs b/wicketd/src/helpers.rs new file mode 100644 index 0000000000..a8b47d4f12 --- /dev/null +++ b/wicketd/src/helpers.rs @@ -0,0 +1,41 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Helpers and utility functions for wicketd. + +use std::fmt; + +use gateway_client::types::{SpIdentifier, SpType}; +use itertools::Itertools; + +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash)] +pub(crate) struct SpIdentifierDisplay(pub(crate) SpIdentifier); + +impl From for SpIdentifierDisplay { + fn from(id: SpIdentifier) -> Self { + SpIdentifierDisplay(id) + } +} + +impl<'a> From<&'a SpIdentifier> for SpIdentifierDisplay { + fn from(id: &'a SpIdentifier) -> Self { + SpIdentifierDisplay(*id) + } +} + +impl fmt::Display for SpIdentifierDisplay { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0.type_ { + SpType::Sled => write!(f, "sled {}", self.0.slot), + SpType::Switch => write!(f, "switch {}", self.0.slot), + SpType::Power => write!(f, "PSC {}", self.0.slot), + } + } +} + +pub(crate) fn sps_to_string>( + sps: impl IntoIterator, +) -> String { + sps.into_iter().map_into().join(", ") +} diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs index 98cac8dc5d..72c3341334 100644 --- a/wicketd/src/http_entrypoints.rs +++ b/wicketd/src/http_entrypoints.rs @@ -4,6 +4,8 @@ //! HTTP entrypoint functions for wicketd +use crate::helpers::sps_to_string; +use crate::helpers::SpIdentifierDisplay; use crate::mgs::GetInventoryError; use crate::mgs::GetInventoryResponse; use crate::mgs::MgsHandle; @@ -44,7 +46,6 @@ use std::net::IpAddr; use std::net::Ipv6Addr; use std::time::Duration; use tokio::io::AsyncWriteExt; -use uuid::Uuid; use wicket_common::rack_setup::PutRssUserConfigInsensitive; use wicket_common::update_events::EventReport; @@ -652,6 +653,15 @@ async fn get_artifacts_and_event_reports( Ok(HttpResponseOk(response)) } +#[derive(Clone, Debug, JsonSchema, Deserialize)] +pub(crate) struct StartUpdateParams { + /// The SP identifiers to start the update with. Must be non-empty. + pub(crate) targets: BTreeSet, + + /// Options for the update. + pub(crate) options: StartUpdateOptions, +} + #[derive(Clone, Debug, JsonSchema, Deserialize)] pub(crate) struct StartUpdateOptions { /// If passed in, fails the update with a simulated error. @@ -730,19 +740,24 @@ impl UpdateTestError { log: &slog::Logger, reason: &str, ) -> HttpError { + let message = self.into_error_string(log, reason).await; + HttpError::for_bad_request(None, message) + } + + pub(crate) async fn into_error_string( + self, + log: &slog::Logger, + reason: &str, + ) -> String { match self { - UpdateTestError::Fail => HttpError::for_bad_request( - None, - format!("Simulated failure while {reason}"), - ), + UpdateTestError::Fail => { + format!("Simulated failure while {reason}") + } UpdateTestError::Timeout { secs } => { slog::info!(log, "Simulating timeout while {reason}"); // 15 seconds should be enough to cause a timeout. tokio::time::sleep(Duration::from_secs(secs)).await; - HttpError::for_bad_request( - None, - "XXX request should time out before this is hit".into(), - ) + "XXX request should time out before this is hit".into() } } } @@ -834,21 +849,27 @@ async fn get_location( })) } -/// An endpoint to start updating a sled. +/// An endpoint to start updating one or more sleds, switches and PSCs. #[endpoint { method = POST, - path = "/update/{type}/{slot}", + path = "/update", }] async fn post_start_update( rqctx: RequestContext, - target: Path, - opts: TypedBody, + params: TypedBody, ) -> Result { let log = &rqctx.log; let rqctx = rqctx.context(); - let target = target.into_inner(); + let params = params.into_inner(); + + if params.targets.is_empty() { + return Err(HttpError::for_bad_request( + None, + "No update targets specified".into(), + )); + } - // Can we update the target SP? We refuse to update if: + // Can we update the target SPs? We refuse to update if, for any target SP: // // 1. We haven't pulled its state in our inventory (most likely cause: the // cubby is empty; less likely cause: the SP is misbehaving, which will @@ -870,70 +891,136 @@ async fn post_start_update( } }; - // Next, do we have the state of the target SP? - let sp_state = match inventory { + // Error cases. + let mut inventory_absent = BTreeSet::new(); + let mut self_update = None; + let mut maybe_self_update = BTreeSet::new(); + + // Next, do we have the states of the target SP? + let sp_states = match inventory { GetInventoryResponse::Response { inventory, .. } => inventory .sps .into_iter() - .filter_map(|sp| if sp.id == target { sp.state } else { None }) - .next(), - GetInventoryResponse::Unavailable => None, - }; - let Some(sp_state) = sp_state else { - return Err(HttpError::for_bad_request( - None, - "cannot update target sled (no inventory state present)".into(), - )); + .filter_map(|sp| { + if params.targets.contains(&sp.id) { + if let Some(sp_state) = sp.state { + Some((sp.id, sp_state)) + } else { + None + } + } else { + None + } + }) + .collect(), + GetInventoryResponse::Unavailable => BTreeMap::new(), }; - // If we have the state of the SP, are we allowed to update it? We - // refuse to try to update our own sled. - match &rqctx.baseboard { - Some(baseboard) => { - if baseboard.identifier() == sp_state.serial_number - && baseboard.model() == sp_state.model - && baseboard.revision() == i64::from(sp_state.revision) - { - return Err(HttpError::for_bad_request( - None, - "cannot update sled where wicketd is running".into(), - )); + for target in ¶ms.targets { + let sp_state = match sp_states.get(target) { + Some(sp_state) => sp_state, + None => { + // The state isn't present, so add to inventory_absent. + inventory_absent.insert(*target); + continue; } - } - None => { - // We don't know our own baseboard, which is a very - // questionable state to be in! For now, we will hard-code - // the possibly locations where we could be running: - // scrimlets can only be in cubbies 14 or 16, so we refuse - // to update either of those. - let target_is_scrimlet = - matches!((target.type_, target.slot), (SpType::Sled, 14 | 16)); - if target_is_scrimlet { - return Err(HttpError::for_bad_request( - None, - "wicketd does not know its own baseboard details: \ - refusing to update either scrimlet" - .into(), - )); + }; + + // If we have the state of the SP, are we allowed to update it? We + // refuse to try to update our own sled. + match &rqctx.baseboard { + Some(baseboard) => { + if baseboard.identifier() == sp_state.serial_number + && baseboard.model() == sp_state.model + && baseboard.revision() == i64::from(sp_state.revision) + { + self_update = Some(*target); + continue; + } + } + None => { + // We don't know our own baseboard, which is a very questionable + // state to be in! For now, we will hard-code the possibly + // locations where we could be running: scrimlets can only be in + // cubbies 14 or 16, so we refuse to update either of those. + let target_is_scrimlet = matches!( + (target.type_, target.slot), + (SpType::Sled, 14 | 16) + ); + if target_is_scrimlet { + maybe_self_update.insert(*target); + continue; + } } } } - let opts = opts.into_inner(); - if let Some(test_error) = opts.test_error { - return Err(test_error.into_http_error(log, "starting update").await); + // Do we have any errors? + let mut errors = Vec::new(); + if !inventory_absent.is_empty() { + errors.push(format!( + "cannot update sleds (no inventory state present for {})", + sps_to_string(&inventory_absent) + )); + } + if let Some(self_update) = self_update { + errors.push(format!( + "cannot update sled where wicketd is running ({})", + SpIdentifierDisplay(self_update) + )); + } + if !maybe_self_update.is_empty() { + errors.push(format!( + "wicketd does not know its own baseboard details: \ + refusing to update either scrimlet ({})", + sps_to_string(&inventory_absent) + )); } - // All pre-flight update checks look OK: start the update. - // - // Generate an ID for this update; the update tracker will send it to the - // sled as part of the InstallinatorImageId, and installinator will send it - // back to our artifact server with its progress reports. - let update_id = Uuid::new_v4(); + if let Some(test_error) = ¶ms.options.test_error { + errors.push(test_error.into_error_string(log, "starting update").await); + } - match rqctx.update_tracker.start(target, update_id, opts).await { - Ok(()) => Ok(HttpResponseUpdatedNoContent {}), - Err(err) => Err(err.to_http_error()), + let start_update_errors = if errors.is_empty() { + // No errors: we can try and proceed with this update. + match rqctx.update_tracker.start(params.targets, params.options).await { + Ok(()) => return Ok(HttpResponseUpdatedNoContent {}), + Err(errors) => errors, + } + } else { + // We've already found errors, so all we want to do is to check whether + // the update tracker thinks there are any errors as well. + match rqctx.update_tracker.update_pre_checks(params.targets).await { + Ok(()) => Vec::new(), + Err(errors) => errors, + } + }; + + errors.extend(start_update_errors.iter().map(|error| error.to_string())); + + // If we get here, we have errors to report. + + match errors.len() { + 0 => { + unreachable!( + "we already returned Ok(_) above if there were no errors" + ) + } + 1 => { + return Err(HttpError::for_bad_request( + None, + errors.pop().unwrap(), + )); + } + _ => { + return Err(HttpError::for_bad_request( + None, + format!( + "multiple errors encountered:\n - {}", + itertools::join(errors, "\n - ") + ), + )); + } } } diff --git a/wicketd/src/lib.rs b/wicketd/src/lib.rs index 78209ea04a..e17c15642c 100644 --- a/wicketd/src/lib.rs +++ b/wicketd/src/lib.rs @@ -6,6 +6,7 @@ mod artifacts; mod bootstrap_addrs; mod config; mod context; +mod helpers; mod http_entrypoints; mod installinator_progress; mod inventory; diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index a95a98bd72..1bbda00158 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -7,6 +7,7 @@ use crate::artifacts::ArtifactIdData; use crate::artifacts::UpdatePlan; use crate::artifacts::WicketdArtifactStore; +use crate::helpers::sps_to_string; use crate::http_entrypoints::GetArtifactsAndEventReportsResponse; use crate::http_entrypoints::StartUpdateOptions; use crate::http_entrypoints::UpdateSimulatedResult; @@ -19,7 +20,6 @@ use anyhow::ensure; use anyhow::Context; use display_error_chain::DisplayErrorChain; use dropshot::HttpError; -use futures::Future; use gateway_client::types::HostPhase2Progress; use gateway_client::types::HostPhase2RecoveryImageId; use gateway_client::types::HostStartupOptions; @@ -156,146 +156,23 @@ impl UpdateTracker { pub(crate) async fn start( &self, - sp: SpIdentifier, - update_id: Uuid, + sps: BTreeSet, opts: StartUpdateOptions, - ) -> Result<(), StartUpdateError> { - self.start_impl(sp, |plan| async { - // Do we need to upload this plan's trampoline phase 2 to MGS? - let upload_trampoline_phase_2_to_mgs = { - let mut upload_trampoline_phase_2_to_mgs = - self.upload_trampoline_phase_2_to_mgs.lock().await; - - match upload_trampoline_phase_2_to_mgs.as_mut() { - Some(prev) => { - // We've previously started an upload - does it match - // this artifact? If not, cancel the old task (which - // might still be trying to upload) and start a new one - // with our current image. - if prev.status.borrow().hash - != plan.trampoline_phase_2.data.hash() - { - // It does _not_ match - we have a new plan with a - // different trampoline image. If the old task is - // still running, cancel it, and start a new one. - prev.task.abort(); - *prev = self - .spawn_upload_trampoline_phase_2_to_mgs(&plan); - } - } - None => { - *upload_trampoline_phase_2_to_mgs = Some( - self.spawn_upload_trampoline_phase_2_to_mgs(&plan), - ); - } - } - - // Both branches above leave `upload_trampoline_phase_2_to_mgs` - // with data, so we can unwrap here to clone the `watch` - // channel. - upload_trampoline_phase_2_to_mgs - .as_ref() - .unwrap() - .status - .clone() - }; - - let event_buffer = Arc::new(StdMutex::new(EventBuffer::new(16))); - let ipr_start_receiver = - self.ipr_update_tracker.register(update_id); - - let update_cx = UpdateContext { - update_id, - sp, - mgs_client: self.mgs_client.clone(), - upload_trampoline_phase_2_to_mgs, - log: self.log.new(o!( - "sp" => format!("{sp:?}"), - "update_id" => update_id.to_string(), - )), - }; - // TODO do we need `UpdateDriver` as a distinct type? - let update_driver = UpdateDriver {}; - - // Using a oneshot channel to communicate the abort handle isn't - // ideal, but it works and is the easiest way to send it without - // restructuring this code. - let (abort_handle_sender, abort_handle_receiver) = - oneshot::channel(); - let task = tokio::spawn(update_driver.run( - plan, - update_cx, - event_buffer.clone(), - ipr_start_receiver, - opts, - abort_handle_sender, - )); - - let abort_handle = abort_handle_receiver - .await - .expect("abort handle is sent immediately"); - - SpUpdateData { task, abort_handle, event_buffer } - }) - .await + ) -> Result<(), Vec> { + let imp = RealSpawnUpdateDriver { update_tracker: self, opts }; + self.start_impl(sps, Some(imp)).await } /// Starts a fake update that doesn't perform any steps, but simply waits - /// for a oneshot receiver to resolve. + /// for a watch receiver to resolve. #[doc(hidden)] pub async fn start_fake_update( &self, - sp: SpIdentifier, - oneshot_receiver: oneshot::Receiver<()>, - ) -> Result<(), StartUpdateError> { - self.start_impl(sp, |_plan| async move { - let (sender, mut receiver) = mpsc::channel(128); - let event_buffer = Arc::new(StdMutex::new(EventBuffer::new(16))); - let event_buffer_2 = event_buffer.clone(); - let log = self.log.clone(); - - let engine = UpdateEngine::new(&log, sender); - let abort_handle = engine.abort_handle(); - - let task = tokio::spawn(async move { - // The step component and ID have been chosen arbitrarily here -- - // they aren't important. - engine - .new_step( - UpdateComponent::Host, - UpdateStepId::RunningInstallinator, - "Fake step that waits for receiver to resolve", - move |_cx| async move { - _ = oneshot_receiver.await; - StepSuccess::new(()).into() - }, - ) - .register(); - - // Spawn a task to accept all events from the executing engine. - let event_receiving_task = tokio::spawn(async move { - while let Some(event) = receiver.recv().await { - event_buffer_2.lock().unwrap().add_event(event); - } - }); - - match engine.execute().await { - Ok(_cx) => (), - Err(err) => { - error!(log, "update failed"; "err" => %err); - } - } - - // Wait for all events to be received and written to the event - // buffer. - event_receiving_task - .await - .expect("event receiving task panicked"); - }); - - SpUpdateData { task, abort_handle, event_buffer } - }) - .await + sps: BTreeSet, + watch_receiver: watch::Receiver<()>, + ) -> Result<(), Vec> { + let imp = FakeUpdateDriver { watch_receiver, log: self.log.clone() }; + self.start_impl(sps, Some(imp)).await } pub(crate) async fn clear_update_state( @@ -315,40 +192,107 @@ impl UpdateTracker { update_data.abort_update(sp, message).await } - async fn start_impl( + /// Checks whether an update can be started for the given SPs, without + /// actually starting it. + /// + /// This should only be used in situations where starting the update is not + /// desired (for example, if we've already encountered errors earlier in the + /// process and we're just adding to the list of errors). In cases where the + /// start method *is* desired, prefer the [`Self::start`] method, which also + /// performs the same checks. + pub(crate) async fn update_pre_checks( &self, - sp: SpIdentifier, - spawn_update_driver: F, - ) -> Result<(), StartUpdateError> + sps: BTreeSet, + ) -> Result<(), Vec> { + self.start_impl::(sps, None).await + } + + async fn start_impl( + &self, + sps: BTreeSet, + spawn_update_driver: Option, + ) -> Result<(), Vec> where - F: FnOnce(UpdatePlan) -> Fut, - Fut: Future + Send, + Spawn: SpawnUpdateDriver, { let mut update_data = self.sp_update_data.lock().await; - let plan = update_data - .artifact_store - .current_plan() - .ok_or(StartUpdateError::TufRepositoryUnavailable)?; + let mut errors = Vec::new(); - match update_data.sp_update_data.entry(sp) { - // Vacant: this is the first time we've started an update to this - // sp. - Entry::Vacant(slot) => { - slot.insert(spawn_update_driver(plan).await); - Ok(()) - } - // Occupied: we've previously started an update to this sp; only - // allow this one if that update is no longer running. - Entry::Occupied(mut slot) => { - if slot.get().task.is_finished() { - slot.insert(spawn_update_driver(plan).await); - Ok(()) - } else { - Err(StartUpdateError::UpdateInProgress(sp)) + // Check that we're not already updating any of these SPs. + let update_in_progress: Vec<_> = sps + .iter() + .filter(|sp| { + // If we don't have any update data for this SP, it's not in + // progress. + // + // If we do, it's in progress if the task is not finished. + update_data + .sp_update_data + .get(sp) + .map_or(false, |data| !data.task.is_finished()) + }) + .copied() + .collect(); + + if !update_in_progress.is_empty() { + errors.push(StartUpdateError::UpdateInProgress(update_in_progress)); + } + + let plan = update_data.artifact_store.current_plan(); + if plan.is_none() { + // (1), referred to below. + errors.push(StartUpdateError::TufRepositoryUnavailable); + } + + // If there are any errors, return now. + if !errors.is_empty() { + return Err(errors); + } + + let plan = + plan.expect("we'd have returned an error at (1) if plan was None"); + + // Call the setup method now. + if let Some(mut spawn_update_driver) = spawn_update_driver { + let setup_data = spawn_update_driver.setup(&plan).await; + + for sp in sps { + match update_data.sp_update_data.entry(sp) { + // Vacant: this is the first time we've started an update to this + // sp. + Entry::Vacant(slot) => { + slot.insert( + spawn_update_driver + .spawn_update_driver( + sp, + plan.clone(), + &setup_data, + ) + .await, + ); + } + // Occupied: we've previously started an update to this sp. + Entry::Occupied(mut slot) => { + assert!( + slot.get().task.is_finished(), + "we just checked that the task was finished" + ); + slot.insert( + spawn_update_driver + .spawn_update_driver( + sp, + plan.clone(), + &setup_data, + ) + .await, + ); + } } } } + + Ok(()) } fn spawn_upload_trampoline_phase_2_to_mgs( @@ -425,6 +369,226 @@ impl UpdateTracker { } } +/// A trait that represents a backend implementation for spawning the update +/// driver. +#[async_trait::async_trait] +trait SpawnUpdateDriver { + /// The type returned by the [`Self::setup`] method. This is passed in by + /// reference to [`Self::spawn_update_driver`]. + type Setup; + + /// Perform setup required to spawn the update driver. + /// + /// This is called *once*, before any calls to + /// [`Self::spawn_update_driver`]. + async fn setup(&mut self, plan: &UpdatePlan) -> Self::Setup; + + /// Spawn the update driver for the given SP. + /// + /// This is called once per SP. + async fn spawn_update_driver( + &mut self, + sp: SpIdentifier, + plan: UpdatePlan, + setup_data: &Self::Setup, + ) -> SpUpdateData; +} + +/// The production implementation of [`SpawnUpdateDriver`]. +/// +/// This implementation spawns real update drivers. +#[derive(Debug)] +struct RealSpawnUpdateDriver<'tr> { + update_tracker: &'tr UpdateTracker, + opts: StartUpdateOptions, +} + +#[async_trait::async_trait] +impl<'tr> SpawnUpdateDriver for RealSpawnUpdateDriver<'tr> { + type Setup = watch::Receiver; + + async fn setup(&mut self, plan: &UpdatePlan) -> Self::Setup { + // Do we need to upload this plan's trampoline phase 2 to MGS? + + let mut upload_trampoline_phase_2_to_mgs = + self.update_tracker.upload_trampoline_phase_2_to_mgs.lock().await; + + match upload_trampoline_phase_2_to_mgs.as_mut() { + Some(prev) => { + // We've previously started an upload - does it match + // this artifact? If not, cancel the old task (which + // might still be trying to upload) and start a new one + // with our current image. + if prev.status.borrow().hash + != plan.trampoline_phase_2.data.hash() + { + // It does _not_ match - we have a new plan with a + // different trampoline image. If the old task is + // still running, cancel it, and start a new one. + prev.task.abort(); + *prev = self + .update_tracker + .spawn_upload_trampoline_phase_2_to_mgs(&plan); + } + } + None => { + *upload_trampoline_phase_2_to_mgs = Some( + self.update_tracker + .spawn_upload_trampoline_phase_2_to_mgs(&plan), + ); + } + } + + // Both branches above leave `upload_trampoline_phase_2_to_mgs` + // with data, so we can unwrap here to clone the `watch` + // channel. + upload_trampoline_phase_2_to_mgs.as_ref().unwrap().status.clone() + } + + async fn spawn_update_driver( + &mut self, + sp: SpIdentifier, + plan: UpdatePlan, + setup_data: &Self::Setup, + ) -> SpUpdateData { + // Generate an ID for this update; the update tracker will send it to the + // sled as part of the InstallinatorImageId, and installinator will send it + // back to our artifact server with its progress reports. + let update_id = Uuid::new_v4(); + + let event_buffer = Arc::new(StdMutex::new(EventBuffer::new(16))); + let ipr_start_receiver = + self.update_tracker.ipr_update_tracker.register(update_id); + + let update_cx = UpdateContext { + update_id, + sp, + mgs_client: self.update_tracker.mgs_client.clone(), + upload_trampoline_phase_2_to_mgs: setup_data.clone(), + log: self.update_tracker.log.new(o!( + "sp" => format!("{sp:?}"), + "update_id" => update_id.to_string(), + )), + }; + // TODO do we need `UpdateDriver` as a distinct type? + let update_driver = UpdateDriver {}; + + // Using a oneshot channel to communicate the abort handle isn't + // ideal, but it works and is the easiest way to send it without + // restructuring this code. + let (abort_handle_sender, abort_handle_receiver) = oneshot::channel(); + let task = tokio::spawn(update_driver.run( + plan, + update_cx, + event_buffer.clone(), + ipr_start_receiver, + self.opts.clone(), + abort_handle_sender, + )); + + let abort_handle = abort_handle_receiver + .await + .expect("abort handle is sent immediately"); + + SpUpdateData { task, abort_handle, event_buffer } + } +} + +/// A fake implementation of [`SpawnUpdateDriver`]. +/// +/// This implementation is only used by tests. It contains a single step that +/// waits for a [`watch::Receiver`] to resolve. +#[derive(Debug)] +struct FakeUpdateDriver { + watch_receiver: watch::Receiver<()>, + log: Logger, +} + +#[async_trait::async_trait] +impl SpawnUpdateDriver for FakeUpdateDriver { + type Setup = (); + + async fn setup(&mut self, _plan: &UpdatePlan) -> Self::Setup {} + + async fn spawn_update_driver( + &mut self, + _sp: SpIdentifier, + _plan: UpdatePlan, + _setup_data: &Self::Setup, + ) -> SpUpdateData { + let (sender, mut receiver) = mpsc::channel(128); + let event_buffer = Arc::new(StdMutex::new(EventBuffer::new(16))); + let event_buffer_2 = event_buffer.clone(); + let log = self.log.clone(); + + let engine = UpdateEngine::new(&log, sender); + let abort_handle = engine.abort_handle(); + + let mut watch_receiver = self.watch_receiver.clone(); + + let task = tokio::spawn(async move { + // The step component and ID have been chosen arbitrarily here -- + // they aren't important. + engine + .new_step( + UpdateComponent::Host, + UpdateStepId::RunningInstallinator, + "Fake step that waits for receiver to resolve", + move |_cx| async move { + // This will resolve as soon as the watch sender + // (typically a test) sends a value over the watch + // channel. + _ = watch_receiver.changed().await; + StepSuccess::new(()).into() + }, + ) + .register(); + + // Spawn a task to accept all events from the executing engine. + let event_receiving_task = tokio::spawn(async move { + while let Some(event) = receiver.recv().await { + event_buffer_2.lock().unwrap().add_event(event); + } + }); + + match engine.execute().await { + Ok(_cx) => (), + Err(err) => { + error!(log, "update failed"; "err" => %err); + } + } + + // Wait for all events to be received and written to the event + // buffer. + event_receiving_task.await.expect("event receiving task panicked"); + }); + + SpUpdateData { task, abort_handle, event_buffer } + } +} + +/// An implementation of [`SpawnUpdateDriver`] that cannot be constructed. +/// +/// This is an uninhabited type (an empty enum), and is only used to provide a +/// type parameter for the [`UpdateTracker::update_pre_checks`] method. +enum NeverUpdateDriver {} + +#[async_trait::async_trait] +impl SpawnUpdateDriver for NeverUpdateDriver { + type Setup = (); + + async fn setup(&mut self, _plan: &UpdatePlan) -> Self::Setup {} + + async fn spawn_update_driver( + &mut self, + _sp: SpIdentifier, + _plan: UpdatePlan, + _setup_data: &Self::Setup, + ) -> SpUpdateData { + unreachable!("this update driver cannot be constructed") + } +} + #[derive(Debug)] struct UpdateTrackerData { artifact_store: WicketdArtifactStore, @@ -518,21 +682,8 @@ impl UpdateTrackerData { pub enum StartUpdateError { #[error("no TUF repository available")] TufRepositoryUnavailable, - #[error("target is already being updated: {0:?}")] - UpdateInProgress(SpIdentifier), -} - -impl StartUpdateError { - pub(crate) fn to_http_error(&self) -> HttpError { - let message = DisplayErrorChain::new(self).to_string(); - - match self { - StartUpdateError::TufRepositoryUnavailable - | StartUpdateError::UpdateInProgress(_) => { - HttpError::for_bad_request(None, message) - } - } - } + #[error("targets are already being updated: {}", sps_to_string(.0))] + UpdateInProgress(Vec), } #[derive(Debug, Clone, Error, Eq, PartialEq)] diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs index a4b330930a..a198068ef3 100644 --- a/wicketd/tests/integration_tests/updates.rs +++ b/wicketd/tests/integration_tests/updates.rs @@ -16,13 +16,13 @@ use omicron_common::{ api::internal::nexus::KnownArtifactKind, update::{ArtifactHashId, ArtifactKind}, }; -use tokio::sync::oneshot; +use tokio::sync::watch; use uuid::Uuid; use wicket_common::update_events::{StepEventKind, UpdateComponent}; use wicketd::{RunningUpdateState, StartUpdateError}; use wicketd_client::types::{ GetInventoryParams, GetInventoryResponse, SpIdentifier, SpType, - StartUpdateOptions, + StartUpdateOptions, StartUpdateParams, }; #[tokio::test] @@ -138,13 +138,11 @@ async fn test_updates() { } // Now, try starting the update on SP 0. + let options = StartUpdateOptions::default(); + let params = StartUpdateParams { targets: vec![target_sp], options }; wicketd_testctx .wicketd_client - .post_start_update( - target_sp.type_, - target_sp.slot, - &StartUpdateOptions::default(), - ) + .post_start_update(¶ms) .await .expect("update started successfully"); @@ -352,12 +350,13 @@ async fn test_update_races() { slot: 0, type_: gateway_client::types::SpType::Sled, }; + let sps: BTreeSet<_> = vec![sp].into_iter().collect(); - let (sender, receiver) = oneshot::channel(); + let (sender, receiver) = watch::channel(()); wicketd_testctx .server .update_tracker - .start_fake_update(sp, receiver) + .start_fake_update(sps.clone(), receiver) .await .expect("start_fake_update successful"); @@ -372,14 +371,18 @@ async fn test_update_races() { // Also try starting another fake update, which should fail -- we don't let // updates be started in the middle of other updates. { - let (_, receiver) = oneshot::channel(); + let (_, receiver) = watch::channel(()); let err = wicketd_testctx .server .update_tracker - .start_fake_update(sp, receiver) + .start_fake_update(sps, receiver) .await .expect_err("start_fake_update failed while update is running"); - assert_eq!(err, StartUpdateError::UpdateInProgress(sp)); + assert_eq!(err.len(), 1, "one error returned: {err:?}"); + assert_eq!( + err.first().unwrap(), + &StartUpdateError::UpdateInProgress(vec![sp]) + ); } // Unblock the update, letting it run to completion. From 2a6ef48917d16fa4d45375998a1c2e5ff1a89136 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 29 Sep 2023 12:24:54 -0700 Subject: [PATCH 05/89] [db] Access DB via connections, not via pool (#4140) - Accesses the DB via connections explicitly, rather than using a pool - This change reduces generics slightly, and simplifies error handling. Callers can deal with "pool errors" when checking out the connection, and can deal with "query errors" separately when issuing the queries themselves. Depends on https://github.com/oxidecomputer/async-bb8-diesel/pull/52 Fixes https://github.com/oxidecomputer/omicron/issues/4132 --- Cargo.lock | 2 +- Cargo.toml | 2 +- dev-tools/omdb/src/bin/omdb/db.rs | 26 +-- nexus/db-macros/src/lookup.rs | 8 +- nexus/db-queries/src/db/collection_attach.rs | 167 ++++++++------- nexus/db-queries/src/db/collection_detach.rs | 114 +++++----- .../src/db/collection_detach_many.rs | 155 +++++++------- nexus/db-queries/src/db/collection_insert.rs | 47 ++-- .../src/db/datastore/address_lot.rs | 52 +++-- .../src/db/datastore/certificate.rs | 16 +- .../src/db/datastore/console_session.rs | 6 +- nexus/db-queries/src/db/datastore/dataset.rs | 22 +- .../src/db/datastore/db_metadata.rs | 27 ++- .../src/db/datastore/device_auth.rs | 16 +- nexus/db-queries/src/db/datastore/disk.rs | 50 ++--- nexus/db-queries/src/db/datastore/dns.rs | 202 +++++++----------- .../src/db/datastore/external_ip.rs | 37 ++-- .../src/db/datastore/identity_provider.rs | 12 +- nexus/db-queries/src/db/datastore/image.rs | 62 +++--- nexus/db-queries/src/db/datastore/instance.rs | 37 ++-- nexus/db-queries/src/db/datastore/ip_pool.rs | 95 ++++---- nexus/db-queries/src/db/datastore/mod.rs | 90 +++++--- .../src/db/datastore/network_interface.rs | 46 ++-- nexus/db-queries/src/db/datastore/oximeter.rs | 20 +- .../src/db/datastore/physical_disk.rs | 20 +- nexus/db-queries/src/db/datastore/project.rs | 42 ++-- nexus/db-queries/src/db/datastore/rack.rs | 59 ++--- nexus/db-queries/src/db/datastore/region.rs | 29 +-- .../src/db/datastore/region_snapshot.rs | 10 +- nexus/db-queries/src/db/datastore/role.rs | 56 ++--- nexus/db-queries/src/db/datastore/saga.rs | 26 +-- nexus/db-queries/src/db/datastore/service.rs | 38 ++-- nexus/db-queries/src/db/datastore/silo.rs | 168 +++++++-------- .../db-queries/src/db/datastore/silo_group.rs | 41 ++-- .../db-queries/src/db/datastore/silo_user.rs | 69 +++--- nexus/db-queries/src/db/datastore/sled.rs | 22 +- .../src/db/datastore/sled_instance.rs | 8 +- nexus/db-queries/src/db/datastore/snapshot.rs | 27 +-- nexus/db-queries/src/db/datastore/ssh_key.rs | 14 +- nexus/db-queries/src/db/datastore/switch.rs | 12 +- .../src/db/datastore/switch_interface.rs | 52 ++--- .../src/db/datastore/switch_port.rs | 127 +++++------ nexus/db-queries/src/db/datastore/update.rs | 62 +++--- .../virtual_provisioning_collection.rs | 80 +++---- nexus/db-queries/src/db/datastore/volume.rs | 111 +++++----- nexus/db-queries/src/db/datastore/vpc.rs | 152 +++++++------ nexus/db-queries/src/db/datastore/zpool.rs | 22 +- nexus/db-queries/src/db/error.rs | 123 +++++------ nexus/db-queries/src/db/explain.rs | 18 +- nexus/db-queries/src/db/lookup.rs | 2 +- nexus/db-queries/src/db/pagination.rs | 21 +- .../db-queries/src/db/queries/external_ip.rs | 35 +-- .../src/db/queries/network_interface.rs | 107 +++++----- nexus/db-queries/src/db/queries/next_item.rs | 9 +- .../src/db/queries/region_allocation.rs | 4 +- nexus/db-queries/src/db/queries/vpc_subnet.rs | 33 ++- nexus/db-queries/src/db/true_or_cast_error.rs | 8 +- nexus/db-queries/src/db/update_and_check.rs | 11 +- nexus/src/app/background/dns_config.rs | 6 +- nexus/src/app/background/dns_servers.rs | 12 +- nexus/src/app/background/init.rs | 2 +- nexus/src/app/sagas/disk_create.rs | 12 +- nexus/src/app/sagas/instance_create.rs | 18 +- nexus/src/app/sagas/project_create.rs | 6 +- nexus/src/app/sagas/snapshot_create.rs | 6 +- nexus/src/app/sagas/test_helpers.rs | 2 +- nexus/src/app/sagas/vpc_create.rs | 16 +- 67 files changed, 1474 insertions(+), 1535 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 138080640e..b7296ea184 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,7 +287,7 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-bb8-diesel" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=be3d9bce50051d8c0e0c06078e8066cc27db3001#be3d9bce50051d8c0e0c06078e8066cc27db3001" +source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=da04c087f835a51e0441addb19c5ef4986e1fcf2#da04c087f835a51e0441addb19c5ef4986e1fcf2" dependencies = [ "async-trait", "bb8", diff --git a/Cargo.toml b/Cargo.toml index d660397d9e..0e194394f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,7 +135,7 @@ api_identity = { path = "api_identity" } approx = "0.5.1" assert_matches = "1.5.0" assert_cmd = "2.0.12" -async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "be3d9bce50051d8c0e0c06078e8066cc27db3001" } +async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "da04c087f835a51e0441addb19c5ef4986e1fcf2" } async-trait = "0.1.73" authz-macros = { path = "nexus/authz-macros" } backoff = { version = "0.4.0", features = [ "tokio" ] } diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 42f4d53730..93e5ef4301 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -367,7 +367,7 @@ async fn cmd_db_disk_list( .filter(dsl::time_deleted.is_null()) .limit(i64::from(u32::from(limit))) .select(Disk::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading disks")?; @@ -421,11 +421,13 @@ async fn cmd_db_disk_info( use db::schema::disk::dsl as disk_dsl; + let conn = datastore.pool_connection_for_tests().await?; + let disk = disk_dsl::disk .filter(disk_dsl::id.eq(args.uuid)) .limit(1) .select(Disk::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*conn) .await .context("loading requested disk")?; @@ -445,7 +447,7 @@ async fn cmd_db_disk_info( .filter(instance_dsl::id.eq(instance_uuid)) .limit(1) .select(Instance::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*conn) .await .context("loading requested instance")?; @@ -540,7 +542,7 @@ async fn cmd_db_disk_physical( .filter(zpool_dsl::time_deleted.is_null()) .filter(zpool_dsl::physical_disk_id.eq(args.uuid)) .select(Zpool::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading zpool from pysical disk id")?; @@ -560,7 +562,7 @@ async fn cmd_db_disk_physical( .filter(dataset_dsl::time_deleted.is_null()) .filter(dataset_dsl::pool_id.eq(zp.id())) .select(Dataset::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading dataset")?; @@ -595,7 +597,7 @@ async fn cmd_db_disk_physical( let regions = region_dsl::region .filter(region_dsl::dataset_id.eq(did)) .select(Region::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading region")?; @@ -614,7 +616,7 @@ async fn cmd_db_disk_physical( .filter(dsl::volume_id.eq_any(volume_ids)) .limit(i64::from(u32::from(limit))) .select(Disk::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading disks")?; @@ -642,7 +644,7 @@ async fn cmd_db_disk_physical( .filter(instance_dsl::id.eq(instance_uuid)) .limit(1) .select(Instance::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading requested instance")?; @@ -877,7 +879,7 @@ async fn cmd_db_instances( let instances = dsl::instance .limit(i64::from(u32::from(limit))) .select(Instance::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading instances")?; @@ -971,7 +973,7 @@ async fn load_zones_version( .filter(dsl::version.eq(nexus_db_model::Generation::from(version))) .limit(1) .select(DnsVersion::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading requested version")?; @@ -1013,7 +1015,7 @@ async fn cmd_db_dns_diff( .filter(dsl::version_added.eq(version.version)) .limit(i64::from(u32::from(limit))) .select(DnsName::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading added names")?; check_limit(&added, limit, || "loading added names"); @@ -1023,7 +1025,7 @@ async fn cmd_db_dns_diff( .filter(dsl::version_removed.eq(version.version)) .limit(i64::from(u32::from(limit))) .select(DnsName::as_select()) - .load_async(datastore.pool_for_tests().await?) + .load_async(&*datastore.pool_connection_for_tests().await?) .await .context("loading added names")?; check_limit(&added, limit, || "loading removed names"); diff --git a/nexus/db-macros/src/lookup.rs b/nexus/db-macros/src/lookup.rs index 93c2bd3652..38cab15e30 100644 --- a/nexus/db-macros/src/lookup.rs +++ b/nexus/db-macros/src/lookup.rs @@ -806,11 +806,11 @@ fn generate_database_functions(config: &Config) -> TokenStream { #lookup_filter .select(nexus_db_model::#resource_name::as_select()) .get_result_async( - datastore.pool_authorized(opctx).await? + &*datastore.pool_connection_authorized(opctx).await? ) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::#resource_name, @@ -891,10 +891,10 @@ fn generate_database_functions(config: &Config) -> TokenStream { #soft_delete_filter #(.filter(dsl::#pkey_column_names.eq(#pkey_names.clone())))* .select(nexus_db_model::#resource_name::as_select()) - .get_result_async(datastore.pool_authorized(opctx).await?) + .get_result_async(&*datastore.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::#resource_name, diff --git a/nexus/db-queries/src/db/collection_attach.rs b/nexus/db-queries/src/db/collection_attach.rs index c88054795d..40ec659bf9 100644 --- a/nexus/db-queries/src/db/collection_attach.rs +++ b/nexus/db-queries/src/db/collection_attach.rs @@ -17,7 +17,7 @@ use super::cte_utils::{ QueryFromClause, QuerySqlType, TableDefaultWhereClause, }; use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, PoolError}; +use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionError}; use diesel::associations::HasTable; use diesel::expression::{AsExpression, Expression}; use diesel::helper_types::*; @@ -299,7 +299,7 @@ where /// Result of [`AttachToCollectionStatement`] when executed asynchronously pub type AsyncAttachToCollectionResult = - Result<(C, ResourceType), AttachError>; + Result<(C, ResourceType), AttachError>; /// Errors returned by [`AttachToCollectionStatement`]. #[derive(Debug)] @@ -332,10 +332,9 @@ where AttachToCollectionStatement: Send, { /// Issues the CTE asynchronously and parses the result. - pub async fn attach_and_get_result_async( + pub async fn attach_and_get_result_async( self, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + conn: &async_bb8_diesel::Connection, ) -> AsyncAttachToCollectionResult where // We require this bound to ensure that "Self" is runnable as query. @@ -344,13 +343,11 @@ where DbConnection, RawOutput, >, - ConnErr: From + Send + 'static, - PoolError: From, { self.get_result_async::>(conn) .await // If the database returns an error, propagate it right away. - .map_err(|e| AttachError::DatabaseError(PoolError::from(e))) + .map_err(|e| AttachError::DatabaseError(e)) // Otherwise, parse the output to determine if the CTE succeeded. .and_then(Self::parse_result) } @@ -570,6 +567,7 @@ mod test { }; use async_bb8_diesel::{ AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, + ConnectionManager, }; use chrono::Utc; use db_macros::Resource; @@ -605,7 +603,9 @@ mod test { } } - async fn setup_db(pool: &crate::db::Pool) { + async fn setup_db( + pool: &crate::db::Pool, + ) -> bb8::PooledConnection> { let connection = pool.pool().get().await.unwrap(); (*connection) .batch_execute_async( @@ -633,6 +633,7 @@ mod test { ) .await .unwrap(); + connection } /// Describes a resource within the database. @@ -669,7 +670,7 @@ mod test { async fn insert_collection( id: Uuid, name: &str, - pool: &db::Pool, + conn: &async_bb8_diesel::Connection, ) -> Collection { let create_params = IdentityMetadataCreateParams { name: Name::try_from(name.to_string()).unwrap(), @@ -680,18 +681,21 @@ mod test { diesel::insert_into(collection::table) .values(c) - .execute_async(pool.pool()) + .execute_async(conn) .await .unwrap(); - get_collection(id, &pool).await + get_collection(id, &conn).await } - async fn get_collection(id: Uuid, pool: &db::Pool) -> Collection { + async fn get_collection( + id: Uuid, + conn: &async_bb8_diesel::Connection, + ) -> Collection { collection::table .find(id) .select(Collection::as_select()) - .first_async(pool.pool()) + .first_async(conn) .await .unwrap() } @@ -699,7 +703,7 @@ mod test { async fn insert_resource( id: Uuid, name: &str, - pool: &db::Pool, + conn: &async_bb8_diesel::Connection, ) -> Resource { let create_params = IdentityMetadataCreateParams { name: Name::try_from(name.to_string()).unwrap(), @@ -712,18 +716,21 @@ mod test { diesel::insert_into(resource::table) .values(r) - .execute_async(pool.pool()) + .execute_async(conn) .await .unwrap(); - get_resource(id, &pool).await + get_resource(id, conn).await } - async fn get_resource(id: Uuid, pool: &db::Pool) -> Resource { + async fn get_resource( + id: Uuid, + conn: &async_bb8_diesel::Connection, + ) -> Resource { resource::table .find(id) .select(Resource::as_select()) - .first_async(pool.pool()) + .first_async(conn) .await .unwrap() } @@ -856,7 +863,7 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); @@ -869,7 +876,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; assert!(matches!(attach, Err(AttachError::CollectionNotFound))); @@ -885,14 +892,14 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection let collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; // Attempt to attach - even though the resource does not exist. let attach = Collection::attach_resource( @@ -904,12 +911,12 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; assert!(matches!(attach, Err(AttachError::ResourceNotFound))); // The collection should remain unchanged. - assert_eq!(collection, get_collection(collection_id, &pool).await); + assert_eq!(collection, get_collection(collection_id, &conn).await); db.cleanup().await.unwrap(); logctx.cleanup_successful(); @@ -922,15 +929,15 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection and resource. let _collection = - insert_collection(collection_id, "collection", &pool).await; - let _resource = insert_resource(resource_id, "resource", &pool).await; + insert_collection(collection_id, "collection", &conn).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; // Attach the resource to the collection. let attach = Collection::attach_resource( @@ -942,7 +949,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; // "attach_and_get_result_async" should return the "attached" resource. @@ -955,9 +962,9 @@ mod test { // The returned value should be the latest value in the DB. assert_eq!( returned_collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); - assert_eq!(returned_resource, get_resource(resource_id, &pool).await); + assert_eq!(returned_resource, get_resource(resource_id, &conn).await); db.cleanup().await.unwrap(); logctx.cleanup_successful(); @@ -970,15 +977,15 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection and resource. let _collection = - insert_collection(collection_id, "collection", &pool).await; - let _resource = insert_resource(resource_id, "resource", &pool).await; + insert_collection(collection_id, "collection", &conn).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; // Attach the resource to the collection. let attach_query = Collection::attach_resource( @@ -991,10 +998,10 @@ mod test { .set(resource::dsl::collection_id.eq(collection_id)), ); - type TxnError = - TransactionError>; - let result = pool - .pool() + type TxnError = TransactionError< + AttachError, + >; + let result = conn .transaction_async(|conn| async move { attach_query.attach_and_get_result_async(&conn).await.map_err( |e| match e { @@ -1015,9 +1022,9 @@ mod test { // The returned values should be the latest value in the DB. assert_eq!( returned_collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); - assert_eq!(returned_resource, get_resource(resource_id, &pool).await); + assert_eq!(returned_resource, get_resource(resource_id, &conn).await); db.cleanup().await.unwrap(); logctx.cleanup_successful(); @@ -1030,7 +1037,7 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; const RESOURCE_COUNT: u32 = 5; @@ -1038,12 +1045,12 @@ mod test { // Create the collection. let _collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; // Create each resource, attaching them to the collection. for i in 0..RESOURCE_COUNT { let resource_id = uuid::Uuid::new_v4(); - insert_resource(resource_id, &format!("resource{}", i), &pool) + insert_resource(resource_id, &format!("resource{}", i), &conn) .await; // Attach the resource to the collection. @@ -1056,7 +1063,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; // "attach_and_get_result_async" should return the "attached" resource. @@ -1071,7 +1078,7 @@ mod test { // The returned resource value should be the latest value in the DB. assert_eq!( returned_resource, - get_resource(resource_id, &pool).await + get_resource(resource_id, &conn).await ); } @@ -1086,15 +1093,15 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); // Attach a resource to a collection, as usual. let _collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; let resource_id1 = uuid::Uuid::new_v4(); - let _resource = insert_resource(resource_id1, "resource1", &pool).await; + let _resource = insert_resource(resource_id1, "resource1", &conn).await; let attach = Collection::attach_resource( collection_id, resource_id1, @@ -1104,7 +1111,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; assert_eq!( attach.expect("Attach should have worked").1.id(), @@ -1113,7 +1120,7 @@ mod test { // Let's try attaching a second resource, now that we're at capacity. let resource_id2 = uuid::Uuid::new_v4(); - let _resource = insert_resource(resource_id2, "resource2", &pool).await; + let _resource = insert_resource(resource_id2, "resource2", &conn).await; let attach = Collection::attach_resource( collection_id, resource_id2, @@ -1123,17 +1130,17 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; let err = attach.expect_err("Should have failed to attach"); match err { AttachError::NoUpdate { attached_count, resource, collection } => { assert_eq!(attached_count, 1); - assert_eq!(resource, get_resource(resource_id2, &pool).await); + assert_eq!(resource, get_resource(resource_id2, &conn).await); assert_eq!( collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); } _ => panic!("Unexpected error: {:?}", err), @@ -1150,15 +1157,15 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); // Attach a resource to a collection, as usual. let _collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; let resource_id = uuid::Uuid::new_v4(); - let _resource = insert_resource(resource_id, "resource", &pool).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; let attach = Collection::attach_resource( collection_id, resource_id, @@ -1168,7 +1175,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; assert_eq!( attach.expect("Attach should have worked").1.id(), @@ -1185,7 +1192,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; let err = attach.expect_err("Should have failed to attach"); @@ -1203,10 +1210,10 @@ mod test { .expect("Should already be attached"), collection_id ); - assert_eq!(resource, get_resource(resource_id, &pool).await); + assert_eq!(resource, get_resource(resource_id, &conn).await); assert_eq!( collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); } _ => panic!("Unexpected error: {:?}", err), @@ -1222,7 +1229,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; let err = attach.expect_err("Should have failed to attach"); // Even when at capacity, the same information should be propagated back @@ -1237,10 +1244,10 @@ mod test { .expect("Should already be attached"), collection_id ); - assert_eq!(resource, get_resource(resource_id, &pool).await); + assert_eq!(resource, get_resource(resource_id, &conn).await); assert_eq!( collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); } _ => panic!("Unexpected error: {:?}", err), @@ -1257,15 +1264,15 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection and resource. let _collection = - insert_collection(collection_id, "collection", &pool).await; - let _resource = insert_resource(resource_id, "resource", &pool).await; + insert_collection(collection_id, "collection", &conn).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; // Attach the resource to the collection. // @@ -1290,7 +1297,7 @@ mod test { resource::dsl::description.eq("new description".to_string()), )), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; let (_, returned_resource) = attach.expect("Attach should have worked"); @@ -1298,7 +1305,7 @@ mod test { returned_resource.collection_id.expect("Expected a collection ID"), collection_id ); - assert_eq!(returned_resource, get_resource(resource_id, &pool).await); + assert_eq!(returned_resource, get_resource(resource_id, &conn).await); assert_eq!(returned_resource.description(), "new description"); db.cleanup().await.unwrap(); @@ -1312,22 +1319,22 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection and resource. let _collection = - insert_collection(collection_id, "collection", &pool).await; - let _resource = insert_resource(resource_id, "resource", &pool).await; + insert_collection(collection_id, "collection", &conn).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; // Immediately soft-delete the resource. diesel::update( resource::table.filter(resource::dsl::id.eq(resource_id)), ) .set(resource::dsl::time_deleted.eq(Utc::now())) - .execute_async(pool.pool()) + .execute_async(&*conn) .await .unwrap(); @@ -1342,7 +1349,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; assert!(matches!(attach, Err(AttachError::ResourceNotFound))); @@ -1357,19 +1364,19 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); // Create the collection and some resources. let _collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; let resource_id1 = uuid::Uuid::new_v4(); let resource_id2 = uuid::Uuid::new_v4(); let _resource1 = - insert_resource(resource_id1, "resource1", &pool).await; + insert_resource(resource_id1, "resource1", &conn).await; let _resource2 = - insert_resource(resource_id2, "resource2", &pool).await; + insert_resource(resource_id2, "resource2", &conn).await; // Attach the resource to the collection. // @@ -1384,7 +1391,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(&conn) .await; let (_, returned_resource) = attach.expect("Attach should have worked"); @@ -1394,10 +1401,10 @@ mod test { // "resource2" should have automatically been filtered away from the // update statement, regardless of user input. assert_eq!( - get_resource(resource_id1, &pool).await.collection_id.unwrap(), + get_resource(resource_id1, &conn).await.collection_id.unwrap(), collection_id ); - assert!(get_resource(resource_id2, &pool) + assert!(get_resource(resource_id2, &conn) .await .collection_id .is_none()); diff --git a/nexus/db-queries/src/db/collection_detach.rs b/nexus/db-queries/src/db/collection_detach.rs index 04894ecb21..df157040e6 100644 --- a/nexus/db-queries/src/db/collection_detach.rs +++ b/nexus/db-queries/src/db/collection_detach.rs @@ -16,7 +16,7 @@ use super::cte_utils::{ QueryFromClause, QuerySqlType, }; use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, PoolError}; +use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionError}; use diesel::associations::HasTable; use diesel::expression::{AsExpression, Expression}; use diesel::helper_types::*; @@ -230,7 +230,7 @@ where /// Result of [`DetachFromCollectionStatement`] when executed asynchronously pub type AsyncDetachFromCollectionResult = - Result>; + Result>; /// Errors returned by [`DetachFromCollectionStatement`]. #[derive(Debug)] @@ -265,8 +265,7 @@ where /// Issues the CTE asynchronously and parses the result. pub async fn detach_and_get_result_async( self, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + conn: &async_bb8_diesel::Connection, ) -> AsyncDetachFromCollectionResult where // We require this bound to ensure that "Self" is runnable as query. @@ -482,7 +481,9 @@ mod test { use super::*; use crate::db::collection_attach::DatastoreAttachTarget; use crate::db::{self, identity::Resource as IdentityResource}; - use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; + use async_bb8_diesel::{ + AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, + }; use chrono::Utc; use db_macros::Resource; use diesel::expression_methods::ExpressionMethods; @@ -517,7 +518,9 @@ mod test { } } - async fn setup_db(pool: &crate::db::Pool) { + async fn setup_db( + pool: &crate::db::Pool, + ) -> bb8::PooledConnection> { let connection = pool.pool().get().await.unwrap(); (*connection) .batch_execute_async( @@ -545,6 +548,7 @@ mod test { ) .await .unwrap(); + connection } /// Describes a resource within the database. @@ -581,7 +585,7 @@ mod test { async fn insert_collection( id: Uuid, name: &str, - pool: &db::Pool, + conn: &async_bb8_diesel::Connection, ) -> Collection { let create_params = IdentityMetadataCreateParams { name: Name::try_from(name.to_string()).unwrap(), @@ -592,18 +596,21 @@ mod test { diesel::insert_into(collection::table) .values(c) - .execute_async(pool.pool()) + .execute_async(conn) .await .unwrap(); - get_collection(id, &pool).await + get_collection(id, conn).await } - async fn get_collection(id: Uuid, pool: &db::Pool) -> Collection { + async fn get_collection( + id: Uuid, + conn: &async_bb8_diesel::Connection, + ) -> Collection { collection::table .find(id) .select(Collection::as_select()) - .first_async(pool.pool()) + .first_async(conn) .await .unwrap() } @@ -611,7 +618,7 @@ mod test { async fn insert_resource( id: Uuid, name: &str, - pool: &db::Pool, + conn: &async_bb8_diesel::Connection, ) -> Resource { let create_params = IdentityMetadataCreateParams { name: Name::try_from(name.to_string()).unwrap(), @@ -624,17 +631,17 @@ mod test { diesel::insert_into(resource::table) .values(r) - .execute_async(pool.pool()) + .execute_async(conn) .await .unwrap(); - get_resource(id, &pool).await + get_resource(id, conn).await } async fn attach_resource( collection_id: Uuid, resource_id: Uuid, - pool: &db::Pool, + conn: &async_bb8_diesel::Connection, ) { Collection::attach_resource( collection_id, @@ -645,16 +652,19 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(conn) .await .unwrap(); } - async fn get_resource(id: Uuid, pool: &db::Pool) -> Resource { + async fn get_resource( + id: Uuid, + conn: &async_bb8_diesel::Connection, + ) -> Resource { resource::table .find(id) .select(Resource::as_select()) - .first_async(pool.pool()) + .first_async(conn) .await .unwrap() } @@ -777,7 +787,7 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); @@ -789,7 +799,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; assert!(matches!(detach, Err(DetachError::CollectionNotFound))); @@ -805,14 +815,14 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection let collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; // Attempt to detach - even though the resource does not exist. let detach = Collection::detach_resource( @@ -823,12 +833,12 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; assert!(matches!(detach, Err(DetachError::ResourceNotFound))); // The collection should remain unchanged. - assert_eq!(collection, get_collection(collection_id, &pool).await); + assert_eq!(collection, get_collection(collection_id, &conn).await); db.cleanup().await.unwrap(); logctx.cleanup_successful(); @@ -841,16 +851,16 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection and resource. Attach them. let _collection = - insert_collection(collection_id, "collection", &pool).await; - let _resource = insert_resource(resource_id, "resource", &pool).await; - attach_resource(collection_id, resource_id, &pool).await; + insert_collection(collection_id, "collection", &conn).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; + attach_resource(collection_id, resource_id, &conn).await; // Detach the resource from the collection. let detach = Collection::detach_resource( @@ -861,14 +871,14 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; // "detach_and_get_result_async" should return the "detached" resource. let returned_resource = detach.expect("Detach should have worked"); assert!(returned_resource.collection_id.is_none(),); // The returned value should be the latest value in the DB. - assert_eq!(returned_resource, get_resource(resource_id, &pool).await); + assert_eq!(returned_resource, get_resource(resource_id, &conn).await); db.cleanup().await.unwrap(); logctx.cleanup_successful(); @@ -881,15 +891,15 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let _collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; let resource_id = uuid::Uuid::new_v4(); - let _resource = insert_resource(resource_id, "resource", &pool).await; - attach_resource(collection_id, resource_id, &pool).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; + attach_resource(collection_id, resource_id, &conn).await; // Detach a resource from a collection, as usual. let detach = Collection::detach_resource( @@ -900,7 +910,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; assert_eq!( detach.expect("Detach should have worked").id(), @@ -916,7 +926,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; let err = detach.expect_err("Should have failed to detach"); @@ -925,10 +935,10 @@ mod test { match err { DetachError::NoUpdate { resource, collection } => { assert!(resource.collection_id.as_ref().is_none()); - assert_eq!(resource, get_resource(resource_id, &pool).await); + assert_eq!(resource, get_resource(resource_id, &conn).await); assert_eq!( collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); } _ => panic!("Unexpected error: {:?}", err), @@ -945,22 +955,22 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection and resource. let _collection = - insert_collection(collection_id, "collection", &pool).await; - let _resource = insert_resource(resource_id, "resource", &pool).await; + insert_collection(collection_id, "collection", &conn).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; // Immediately soft-delete the resource. diesel::update( resource::table.filter(resource::dsl::id.eq(resource_id)), ) .set(resource::dsl::time_deleted.eq(Utc::now())) - .execute_async(pool.pool()) + .execute_async(&*conn) .await .unwrap(); @@ -974,7 +984,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; assert!(matches!(detach, Err(DetachError::ResourceNotFound))); @@ -989,21 +999,21 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); // Create the collection and some resources. let _collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; let resource_id1 = uuid::Uuid::new_v4(); let resource_id2 = uuid::Uuid::new_v4(); let _resource1 = - insert_resource(resource_id1, "resource1", &pool).await; - attach_resource(collection_id, resource_id1, &pool).await; + insert_resource(resource_id1, "resource1", &conn).await; + attach_resource(collection_id, resource_id1, &conn).await; let _resource2 = - insert_resource(resource_id2, "resource2", &pool).await; - attach_resource(collection_id, resource_id2, &pool).await; + insert_resource(resource_id2, "resource2", &conn).await; + attach_resource(collection_id, resource_id2, &conn).await; // Detach the resource from the collection. // @@ -1017,7 +1027,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; let returned_resource = detach.expect("Detach should have worked"); @@ -1026,11 +1036,11 @@ mod test { // Note that only "resource1" should be detached. // "resource2" should have automatically been filtered away from the // update statement, regardless of user input. - assert!(get_resource(resource_id1, &pool) + assert!(get_resource(resource_id1, &conn) .await .collection_id .is_none()); - assert!(get_resource(resource_id2, &pool) + assert!(get_resource(resource_id2, &conn) .await .collection_id .is_some()); diff --git a/nexus/db-queries/src/db/collection_detach_many.rs b/nexus/db-queries/src/db/collection_detach_many.rs index 3418296568..0b65c404c5 100644 --- a/nexus/db-queries/src/db/collection_detach_many.rs +++ b/nexus/db-queries/src/db/collection_detach_many.rs @@ -16,7 +16,7 @@ use super::cte_utils::{ QueryFromClause, QuerySqlType, }; use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, PoolError}; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::associations::HasTable; use diesel::expression::{AsExpression, Expression}; use diesel::helper_types::*; @@ -241,7 +241,7 @@ where /// Result of [`DetachManyFromCollectionStatement`] when executed asynchronously pub type AsyncDetachManyFromCollectionResult = - Result>; + Result>; /// Errors returned by [`DetachManyFromCollectionStatement`]. #[derive(Debug)] @@ -273,21 +273,18 @@ where DetachManyFromCollectionStatement: Send, { /// Issues the CTE asynchronously and parses the result. - pub async fn detach_and_get_result_async( + pub async fn detach_and_get_result_async( self, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + conn: &async_bb8_diesel::Connection, ) -> AsyncDetachManyFromCollectionResult where // We require this bound to ensure that "Self" is runnable as query. Self: query_methods::LoadQuery<'static, DbConnection, RawOutput>, - ConnErr: From + Send + 'static, - PoolError: From, { self.get_result_async::>(conn) .await // If the database returns an error, propagate it right away. - .map_err(|e| DetachManyError::DatabaseError(PoolError::from(e))) + .map_err(|e| DetachManyError::DatabaseError(e)) // Otherwise, parse the output to determine if the CTE succeeded. .and_then(Self::parse_result) } @@ -486,6 +483,7 @@ mod test { }; use async_bb8_diesel::{ AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, + ConnectionManager, }; use chrono::Utc; use db_macros::Resource; @@ -521,7 +519,9 @@ mod test { } } - async fn setup_db(pool: &crate::db::Pool) { + async fn setup_db( + pool: &crate::db::Pool, + ) -> bb8::PooledConnection> { let connection = pool.pool().get().await.unwrap(); (*connection) .batch_execute_async( @@ -549,6 +549,7 @@ mod test { ) .await .unwrap(); + connection } /// Describes a resource within the database. @@ -585,7 +586,7 @@ mod test { async fn insert_collection( id: Uuid, name: &str, - pool: &db::Pool, + conn: &async_bb8_diesel::Connection, ) -> Collection { let create_params = IdentityMetadataCreateParams { name: Name::try_from(name.to_string()).unwrap(), @@ -596,18 +597,21 @@ mod test { diesel::insert_into(collection::table) .values(c) - .execute_async(pool.pool()) + .execute_async(conn) .await .unwrap(); - get_collection(id, &pool).await + get_collection(id, conn).await } - async fn get_collection(id: Uuid, pool: &db::Pool) -> Collection { + async fn get_collection( + id: Uuid, + conn: &async_bb8_diesel::Connection, + ) -> Collection { collection::table .find(id) .select(Collection::as_select()) - .first_async(pool.pool()) + .first_async(conn) .await .unwrap() } @@ -615,7 +619,7 @@ mod test { async fn insert_resource( id: Uuid, name: &str, - pool: &db::Pool, + conn: &async_bb8_diesel::Connection, ) -> Resource { let create_params = IdentityMetadataCreateParams { name: Name::try_from(name.to_string()).unwrap(), @@ -628,17 +632,17 @@ mod test { diesel::insert_into(resource::table) .values(r) - .execute_async(pool.pool()) + .execute_async(conn) .await .unwrap(); - get_resource(id, &pool).await + get_resource(id, conn).await } async fn attach_resource( collection_id: Uuid, resource_id: Uuid, - pool: &db::Pool, + conn: &async_bb8_diesel::Connection, ) { Collection::attach_resource( collection_id, @@ -649,16 +653,19 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .attach_and_get_result_async(pool.pool()) + .attach_and_get_result_async(conn) .await .unwrap(); } - async fn get_resource(id: Uuid, pool: &db::Pool) -> Resource { + async fn get_resource( + id: Uuid, + conn: &async_bb8_diesel::Connection, + ) -> Resource { resource::table .find(id) .select(Resource::as_select()) - .first_async(pool.pool()) + .first_async(conn) .await .unwrap() } @@ -775,7 +782,7 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let _resource_id = uuid::Uuid::new_v4(); @@ -788,7 +795,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; assert!(matches!(detach, Err(DetachManyError::CollectionNotFound))); @@ -805,14 +812,14 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let _resource_id = uuid::Uuid::new_v4(); // Create the collection let _collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; // Attempt to detach - even though the resource does not exist. let detach = Collection::detach_resources( @@ -824,7 +831,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; let returned_collection = detach.expect("Detach should have worked"); @@ -832,7 +839,7 @@ mod test { // The collection should still be updated. assert_eq!( returned_collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); db.cleanup().await.unwrap(); @@ -846,16 +853,16 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection and resource. Attach them. let _collection = - insert_collection(collection_id, "collection", &pool).await; - let _resource = insert_resource(resource_id, "resource", &pool).await; - attach_resource(collection_id, resource_id, &pool).await; + insert_collection(collection_id, "collection", &conn).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; + attach_resource(collection_id, resource_id, &conn).await; // Detach the resource from the collection. let detach = Collection::detach_resources( @@ -867,7 +874,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; // "detach_and_get_result_async" should return the updated collection. @@ -875,7 +882,7 @@ mod test { // The returned value should be the latest value in the DB. assert_eq!( returned_collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); db.cleanup().await.unwrap(); @@ -889,16 +896,16 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection and resource. let _collection = - insert_collection(collection_id, "collection", &pool).await; - let _resource = insert_resource(resource_id, "resource", &pool).await; - attach_resource(collection_id, resource_id, &pool).await; + insert_collection(collection_id, "collection", &conn).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; + attach_resource(collection_id, resource_id, &conn).await; // Detach the resource from the collection. let detach_query = Collection::detach_resources( @@ -911,10 +918,10 @@ mod test { .set(resource::dsl::collection_id.eq(Option::::None)), ); - type TxnError = - TransactionError>; - let result = pool - .pool() + type TxnError = TransactionError< + DetachManyError, + >; + let result = conn .transaction_async(|conn| async move { detach_query.detach_and_get_result_async(&conn).await.map_err( |e| match e { @@ -930,7 +937,7 @@ mod test { // The returned values should be the latest value in the DB. assert_eq!( returned_collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); db.cleanup().await.unwrap(); @@ -944,15 +951,15 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let _collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; let resource_id = uuid::Uuid::new_v4(); - let _resource = insert_resource(resource_id, "resource", &pool).await; - attach_resource(collection_id, resource_id, &pool).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; + attach_resource(collection_id, resource_id, &conn).await; // Detach a resource from a collection, as usual. let detach = Collection::detach_resources( @@ -964,7 +971,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; assert_eq!( detach.expect("Detach should have worked").description(), @@ -982,7 +989,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; assert_eq!( detach.expect("Detach should have worked").description(), @@ -1000,15 +1007,15 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let _collection = - insert_collection(collection_id, "collection", &pool).await; + insert_collection(collection_id, "collection", &conn).await; let resource_id = uuid::Uuid::new_v4(); - let _resource = insert_resource(resource_id, "resource", &pool).await; - attach_resource(collection_id, resource_id, &pool).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; + attach_resource(collection_id, resource_id, &conn).await; // Detach a resource from a collection, but do so with a picky filter // on the collectipon. @@ -1023,7 +1030,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; let err = detach.expect_err("Expected this detach to fail"); @@ -1034,7 +1041,7 @@ mod test { DetachManyError::NoUpdate { collection } => { assert_eq!( collection, - get_collection(collection_id, &pool).await + get_collection(collection_id, &conn).await ); } _ => panic!("Unexpected error: {:?}", err), @@ -1051,23 +1058,23 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); // Create the collection and resource. let _collection = - insert_collection(collection_id, "collection", &pool).await; - let _resource = insert_resource(resource_id, "resource", &pool).await; - attach_resource(collection_id, resource_id, &pool).await; + insert_collection(collection_id, "collection", &conn).await; + let _resource = insert_resource(resource_id, "resource", &conn).await; + attach_resource(collection_id, resource_id, &conn).await; // Immediately soft-delete the resource. diesel::update( resource::table.filter(resource::dsl::id.eq(resource_id)), ) .set(resource::dsl::time_deleted.eq(Utc::now())) - .execute_async(pool.pool()) + .execute_async(&*conn) .await .unwrap(); @@ -1082,7 +1089,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(collection_id)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; assert_eq!( @@ -1090,7 +1097,7 @@ mod test { "Updated desc" ); assert_eq!( - get_resource(resource_id, &pool) + get_resource(resource_id, &conn) .await .collection_id .as_ref() @@ -1109,20 +1116,20 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; // Create the collection and some resources. let collection_id1 = uuid::Uuid::new_v4(); let _collection1 = - insert_collection(collection_id1, "collection", &pool).await; + insert_collection(collection_id1, "collection", &conn).await; let resource_id1 = uuid::Uuid::new_v4(); let resource_id2 = uuid::Uuid::new_v4(); let _resource1 = - insert_resource(resource_id1, "resource1", &pool).await; - attach_resource(collection_id1, resource_id1, &pool).await; + insert_resource(resource_id1, "resource1", &conn).await; + attach_resource(collection_id1, resource_id1, &conn).await; let _resource2 = - insert_resource(resource_id2, "resource2", &pool).await; - attach_resource(collection_id1, resource_id2, &pool).await; + insert_resource(resource_id2, "resource2", &conn).await; + attach_resource(collection_id1, resource_id2, &conn).await; // Create a separate collection with a resource. // @@ -1130,11 +1137,11 @@ mod test { // on "collection_id1". let collection_id2 = uuid::Uuid::new_v4(); let _collection2 = - insert_collection(collection_id2, "collection2", &pool).await; + insert_collection(collection_id2, "collection2", &conn).await; let resource_id3 = uuid::Uuid::new_v4(); let _resource3 = - insert_resource(resource_id3, "resource3", &pool).await; - attach_resource(collection_id2, resource_id3, &pool).await; + insert_resource(resource_id3, "resource3", &conn).await; + attach_resource(collection_id2, resource_id3, &conn).await; // Detach the resource from the collection. let detach = Collection::detach_resources( @@ -1146,7 +1153,7 @@ mod test { diesel::update(resource::table) .set(resource::dsl::collection_id.eq(Option::::None)), ) - .detach_and_get_result_async(pool.pool()) + .detach_and_get_result_async(&conn) .await; let returned_resource = detach.expect("Detach should have worked"); @@ -1154,18 +1161,18 @@ mod test { assert_eq!(returned_resource.description(), "Updated desc"); // Note that only "resource1" and "resource2" should be detached. - assert!(get_resource(resource_id1, &pool) + assert!(get_resource(resource_id1, &conn) .await .collection_id .is_none()); - assert!(get_resource(resource_id2, &pool) + assert!(get_resource(resource_id2, &conn) .await .collection_id .is_none()); // "resource3" should have been left alone. assert_eq!( - get_resource(resource_id3, &pool) + get_resource(resource_id3, &conn) .await .collection_id .as_ref() diff --git a/nexus/db-queries/src/db/collection_insert.rs b/nexus/db-queries/src/db/collection_insert.rs index cebb21a96d..993f16e048 100644 --- a/nexus/db-queries/src/db/collection_insert.rs +++ b/nexus/db-queries/src/db/collection_insert.rs @@ -10,7 +10,7 @@ //! 3) inserts the child resource row use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionError, PoolError}; +use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionError}; use diesel::associations::HasTable; use diesel::helper_types::*; use diesel::pg::Pg; @@ -170,7 +170,7 @@ pub enum AsyncInsertError { /// The collection that the query was inserting into does not exist CollectionNotFound, /// Other database error - DatabaseError(PoolError), + DatabaseError(ConnectionError), } impl InsertIntoCollectionStatement @@ -188,20 +188,17 @@ where /// - Ok(new row) /// - Error(collection not found) /// - Error(other diesel error) - pub async fn insert_and_get_result_async( + pub async fn insert_and_get_result_async( self, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + conn: &async_bb8_diesel::Connection, ) -> AsyncInsertIntoCollectionResult where // We require this bound to ensure that "Self" is runnable as query. Self: query_methods::LoadQuery<'static, DbConnection, ResourceType>, - ConnErr: From + Send + 'static, - PoolError: From, { self.get_result_async::(conn) .await - .map_err(|e| Self::translate_async_error(PoolError::from(e))) + .map_err(|e| Self::translate_async_error(e)) } /// Issues the CTE asynchronously and parses the result. @@ -210,20 +207,17 @@ where /// - Ok(Vec of new rows) /// - Error(collection not found) /// - Error(other diesel error) - pub async fn insert_and_get_results_async( + pub async fn insert_and_get_results_async( self, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + conn: &async_bb8_diesel::Connection, ) -> AsyncInsertIntoCollectionResult> where // We require this bound to ensure that "Self" is runnable as query. Self: query_methods::LoadQuery<'static, DbConnection, ResourceType>, - ConnErr: From + Send + 'static, - PoolError: From, { self.get_results_async::(conn) .await - .map_err(|e| Self::translate_async_error(PoolError::from(e))) + .map_err(|e| Self::translate_async_error(e)) } /// Check for the intentional division by zero error @@ -244,9 +238,9 @@ where /// Translate from diesel errors into AsyncInsertError, handling the /// intentional division-by-zero error in the CTE. - fn translate_async_error(err: PoolError) -> AsyncInsertError { + fn translate_async_error(err: ConnectionError) -> AsyncInsertError { match err { - PoolError::Connection(ConnectionError::Query(err)) + ConnectionError::Query(err) if Self::error_is_division_by_zero(&err) => { AsyncInsertError::CollectionNotFound @@ -393,7 +387,9 @@ where mod test { use super::*; use crate::db::{self, identity::Resource as IdentityResource}; - use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; + use async_bb8_diesel::{ + AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, + }; use chrono::{NaiveDateTime, TimeZone, Utc}; use db_macros::Resource; use diesel::expression_methods::ExpressionMethods; @@ -426,7 +422,9 @@ mod test { } } - async fn setup_db(pool: &crate::db::Pool) { + async fn setup_db( + pool: &crate::db::Pool, + ) -> bb8::PooledConnection> { let connection = pool.pool().get().await.unwrap(); (*connection) .batch_execute_async( @@ -452,6 +450,7 @@ mod test { ) .await .unwrap(); + connection } /// Describes an organization within the database. @@ -548,7 +547,7 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); @@ -563,7 +562,7 @@ mod test { resource::dsl::collection_id.eq(collection_id), )), ) - .insert_and_get_result_async(pool.pool()) + .insert_and_get_result_async(&conn) .await; assert!(matches!(insert, Err(AsyncInsertError::CollectionNotFound))); @@ -578,7 +577,7 @@ mod test { let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); - setup_db(&pool).await; + let conn = setup_db(&pool).await; let collection_id = uuid::Uuid::new_v4(); let resource_id = uuid::Uuid::new_v4(); @@ -593,7 +592,7 @@ mod test { collection::dsl::time_modified.eq(Utc::now()), collection::dsl::rcgen.eq(1), )]) - .execute_async(pool.pool()) + .execute_async(&*conn) .await .unwrap(); @@ -614,7 +613,7 @@ mod test { resource::dsl::collection_id.eq(collection_id), )]), ) - .insert_and_get_result_async(pool.pool()) + .insert_and_get_result_async(&conn) .await .unwrap(); assert_eq!(resource.id(), resource_id); @@ -627,7 +626,7 @@ mod test { let collection_rcgen = collection::table .find(collection_id) .select(collection::dsl::rcgen) - .first_async::(pool.pool()) + .first_async::(&*conn) .await .unwrap(); diff --git a/nexus/db-queries/src/db/datastore/address_lot.rs b/nexus/db-queries/src/db/datastore/address_lot.rs index 35b45753e6..9d264dbf6b 100644 --- a/nexus/db-queries/src/db/datastore/address_lot.rs +++ b/nexus/db-queries/src/db/datastore/address_lot.rs @@ -7,14 +7,14 @@ use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::datastore::PgConnection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::model::Name; use crate::db::model::{AddressLot, AddressLotBlock, AddressLotReservedBlock}; use crate::db::pagination::paginated; use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, Connection, ConnectionError, PoolError, + AsyncConnection, AsyncRunQueryDsl, Connection, ConnectionError, }; use chrono::Utc; use diesel::result::Error as DieselError; @@ -47,7 +47,7 @@ impl DataStore { use db::schema::address_lot::dsl as lot_dsl; use db::schema::address_lot_block::dsl as block_dsl; - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage @@ -84,16 +84,16 @@ impl DataStore { }) .await .map_err(|e| match e { - PoolError::Connection(ConnectionError::Query( - DieselError::DatabaseError(_, _), - )) => public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::AddressLot, - ¶ms.identity.name.as_str(), - ), - ), - _ => public_error_from_diesel_pool(e, ErrorHandler::Server), + ConnectionError::Query(DieselError::DatabaseError(_, _)) => { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AddressLot, + ¶ms.identity.name.as_str(), + ), + ) + } + _ => public_error_from_diesel(e, ErrorHandler::Server), }) } @@ -110,7 +110,7 @@ impl DataStore { let id = authz_address_lot.id(); - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; #[derive(Debug)] enum AddressLotDeleteError { @@ -121,7 +121,7 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { let rsvd: Vec = rsvd_block_dsl::address_lot_rsvd_block .filter(rsvd_block_dsl::address_lot_id.eq(id)) @@ -151,8 +151,8 @@ impl DataStore { }) .await .map_err(|e| match e { - TxnError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TxnError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } TxnError::CustomError(AddressLotDeleteError::LotInUse) => { Error::invalid_request("lot is in use") @@ -179,9 +179,9 @@ impl DataStore { } .filter(dsl::time_deleted.is_null()) .select(AddressLot::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn address_lot_block_list( @@ -192,14 +192,14 @@ impl DataStore { ) -> ListResultVec { use db::schema::address_lot_block::dsl; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; paginated(dsl::address_lot_block, dsl::id, &pagparams) .filter(dsl::address_lot_id.eq(authz_address_lot.id())) .select(AddressLotBlock::as_select()) - .load_async(pool) + .load_async(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn address_lot_id_for_block_id( @@ -207,7 +207,7 @@ impl DataStore { opctx: &OpContext, address_lot_block_id: Uuid, ) -> LookupResult { - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; use db::schema::address_lot_block; use db::schema::address_lot_block::dsl as block_dsl; @@ -216,11 +216,9 @@ impl DataStore { .filter(address_lot_block::id.eq(address_lot_block_id)) .select(address_lot_block::address_lot_id) .limit(1) - .first_async::(pool) + .first_async::(&*conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; Ok(address_lot_id) } diff --git a/nexus/db-queries/src/db/datastore/certificate.rs b/nexus/db-queries/src/db/datastore/certificate.rs index c37d026251..4b043becd8 100644 --- a/nexus/db-queries/src/db/datastore/certificate.rs +++ b/nexus/db-queries/src/db/datastore/certificate.rs @@ -8,7 +8,7 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::Certificate; use crate::db::model::Name; @@ -49,10 +49,10 @@ impl DataStore { .do_update() .set(dsl::time_modified.eq(dsl::time_modified)) .returning(Certificate::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::Certificate, @@ -117,9 +117,11 @@ impl DataStore { query .select(Certificate::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn certificate_delete( @@ -136,10 +138,10 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::id.eq(authz_cert.id())) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_cert), ) diff --git a/nexus/db-queries/src/db/datastore/console_session.rs b/nexus/db-queries/src/db/datastore/console_session.rs index 1e02f9b61d..113a316ae4 100644 --- a/nexus/db-queries/src/db/datastore/console_session.rs +++ b/nexus/db-queries/src/db/datastore/console_session.rs @@ -46,7 +46,7 @@ impl DataStore { diesel::insert_into(dsl::console_session) .values(session) .returning(ConsoleSession::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { Error::internal_error(&format!( @@ -68,7 +68,7 @@ impl DataStore { .filter(dsl::token.eq(authz_session.id())) .set((dsl::time_last_used.eq(Utc::now()),)) .returning(ConsoleSession::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { Error::internal_error(&format!( @@ -130,7 +130,7 @@ impl DataStore { diesel::delete(dsl::console_session) .filter(dsl::silo_user_id.eq(silo_user_id)) .filter(dsl::token.eq(authz_session.id())) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map(|_rows_deleted| ()) .map_err(|e| { diff --git a/nexus/db-queries/src/db/datastore/dataset.rs b/nexus/db-queries/src/db/datastore/dataset.rs index 55259e922f..99972459c8 100644 --- a/nexus/db-queries/src/db/datastore/dataset.rs +++ b/nexus/db-queries/src/db/datastore/dataset.rs @@ -8,7 +8,7 @@ use super::DataStore; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::identity::Asset; use crate::db::model::Dataset; @@ -44,22 +44,22 @@ impl DataStore { dsl::kind.eq(excluded(dsl::kind)), )), ) - .insert_and_get_result_async(self.pool()) + .insert_and_get_result_async( + &*self.pool_connection_unauthorized().await?, + ) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { type_name: ResourceType::Zpool, lookup_type: LookupType::ById(zpool_id), }, - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::Dataset, - &dataset.id().to_string(), - ), - ) - } + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Dataset, + &dataset.id().to_string(), + ), + ), }) } } diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs index ac43081601..181b3c1798 100644 --- a/nexus/db-queries/src/db/datastore/db_metadata.rs +++ b/nexus/db-queries/src/db/datastore/db_metadata.rs @@ -6,7 +6,7 @@ use super::DataStore; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::TransactionError; use async_bb8_diesel::{ @@ -270,11 +270,9 @@ impl DataStore { let version: String = dsl::db_metadata .filter(dsl::singleton.eq(true)) .select(dsl::version) - .get_result_async(self.pool()) + .get_result_async(&*self.pool_connection_unauthorized().await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; SemverVersion::from_str(&version).map_err(|e| { Error::internal_error(&format!("Invalid schema version: {e}")) @@ -312,9 +310,9 @@ impl DataStore { dsl::time_modified.eq(Utc::now()), dsl::target_version.eq(Some(to_version.to_string())), )) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_unauthorized().await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if rows_updated != 1 { return Err(Error::internal_error( @@ -332,7 +330,7 @@ impl DataStore { target: &SemverVersion, sql: &String, ) -> Result<(), Error> { - let result = self.pool().transaction_async(|conn| async move { + let result = self.pool_connection_unauthorized().await?.transaction_async(|conn| async move { if target.to_string() != EARLIEST_SUPPORTED_VERSION { let validate_version_query = format!("SELECT CAST(\ IF(\ @@ -353,8 +351,8 @@ impl DataStore { match result { Ok(()) => Ok(()), Err(TransactionError::CustomError(())) => panic!("No custom error"), - Err(TransactionError::Pool(e)) => { - Err(public_error_from_diesel_pool(e, ErrorHandler::Server)) + Err(TransactionError::Connection(e)) => { + Err(public_error_from_diesel(e, ErrorHandler::Server)) } } } @@ -378,9 +376,9 @@ impl DataStore { dsl::version.eq(to_version.to_string()), dsl::target_version.eq(None as Option), )) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_unauthorized().await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if rows_updated != 1 { return Err(Error::internal_error( @@ -432,6 +430,7 @@ mod test { let cfg = db::Config { url: crdb.pg_config().clone() }; let pool = Arc::new(db::Pool::new(&logctx.log, &cfg)); + let conn = pool.pool().get().await.unwrap(); // Mimic the layout of "schema/crdb". let config_dir = tempfile::TempDir::new().unwrap(); @@ -457,7 +456,7 @@ mod test { use db::schema::db_metadata::dsl; diesel::update(dsl::db_metadata.filter(dsl::singleton.eq(true))) .set(dsl::version.eq(v0.to_string())) - .execute_async(pool.pool()) + .execute_async(&*conn) .await .expect("Failed to set version back to 0.0.0"); @@ -507,7 +506,7 @@ mod test { "EXISTS (SELECT * FROM pg_tables WHERE tablename = 'widget')" ) ) - .get_result_async::(datastore.pool()) + .get_result_async::(&*datastore.pool_connection_for_tests().await.unwrap()) .await .expect("Failed to query for table"); assert_eq!(result, false, "The 'widget' table should have been deleted, but it exists.\ diff --git a/nexus/db-queries/src/db/datastore/device_auth.rs b/nexus/db-queries/src/db/datastore/device_auth.rs index 62e54f2321..e084834833 100644 --- a/nexus/db-queries/src/db/datastore/device_auth.rs +++ b/nexus/db-queries/src/db/datastore/device_auth.rs @@ -8,7 +8,7 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::model::DeviceAccessToken; @@ -42,9 +42,9 @@ impl DataStore { diesel::insert_into(dsl::device_auth_request) .values(auth_request) .returning(DeviceAuthRequest::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Remove the device authorization request and create a new device @@ -77,7 +77,7 @@ impl DataStore { } type TxnError = TransactionError; - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { match delete_request.execute_async(&conn).await? { @@ -103,8 +103,8 @@ impl DataStore { TxnError::CustomError(TokenGrantError::TooManyRequests) => { Error::internal_error("unexpectedly found multiple device auth requests for the same user code") } - TxnError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TxnError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } }) } @@ -127,10 +127,10 @@ impl DataStore { .filter(dsl::client_id.eq(client_id)) .filter(dsl::device_code.eq(device_code)) .select(DeviceAccessToken::as_select()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::DeviceAccessToken, diff --git a/nexus/db-queries/src/db/datastore/disk.rs b/nexus/db-queries/src/db/datastore/disk.rs index 7ae9967285..80f72c1e18 100644 --- a/nexus/db-queries/src/db/datastore/disk.rs +++ b/nexus/db-queries/src/db/datastore/disk.rs @@ -15,7 +15,7 @@ use crate::db::collection_detach::DatastoreDetachTarget; use crate::db::collection_detach::DetachError; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::identity::Resource; use crate::db::lookup::LookupPath; @@ -71,9 +71,9 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::attach_instance_id.eq(authz_instance.id())) .select(Disk::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn project_create_disk( @@ -98,16 +98,16 @@ impl DataStore { .do_update() .set(dsl::time_modified.eq(dsl::time_modified)), ) - .insert_and_get_result_async(self.pool_authorized(opctx).await?) + .insert_and_get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => authz_project.not_found(), - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Conflict(ResourceType::Disk, name.as_str()), - ) - } + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict(ResourceType::Disk, name.as_str()), + ), })?; let runtime = disk.runtime(); @@ -146,9 +146,9 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::project_id.eq(authz_project.id())) .select(Disk::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Attaches a disk to an instance, if both objects: @@ -199,7 +199,7 @@ impl DataStore { diesel::update(disk::dsl::disk).set(attach_update), ); - let (instance, disk) = query.attach_and_get_result_async(self.pool_authorized(opctx).await?) + let (instance, disk) = query.attach_and_get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .or_else(|e| { match e { @@ -278,7 +278,7 @@ impl DataStore { } }, AttachError::DatabaseError(e) => { - Err(public_error_from_diesel_pool(e, ErrorHandler::Server)) + Err(public_error_from_diesel(e, ErrorHandler::Server)) }, } })?; @@ -331,7 +331,7 @@ impl DataStore { disk::dsl::slot.eq(Option::::None) )) ) - .detach_and_get_result_async(self.pool_authorized(opctx).await?) + .detach_and_get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .or_else(|e| { match e { @@ -405,7 +405,7 @@ impl DataStore { } }, DetachError::DatabaseError(e) => { - Err(public_error_from_diesel_pool(e, ErrorHandler::Server)) + Err(public_error_from_diesel(e, ErrorHandler::Server)) }, } })?; @@ -438,14 +438,14 @@ impl DataStore { .filter(dsl::state_generation.lt(new_runtime.gen)) .set(new_runtime.clone()) .check_if_exists::(disk_id) - .execute_and_check(self.pool()) + .execute_and_check(&*self.pool_connection_authorized(opctx).await?) .await .map(|r| match r.status { UpdateStatus::Updated => true, UpdateStatus::NotUpdatedButExists => false, }) .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_disk), ) @@ -469,14 +469,14 @@ impl DataStore { .filter(dsl::id.eq(disk_id)) .set(dsl::pantry_address.eq(pantry_address.to_string())) .check_if_exists::(disk_id) - .execute_and_check(self.pool_authorized(opctx).await?) + .execute_and_check(&*self.pool_connection_authorized(opctx).await?) .await .map(|r| match r.status { UpdateStatus::Updated => true, UpdateStatus::NotUpdatedButExists => false, }) .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_disk), ) @@ -499,14 +499,14 @@ impl DataStore { .filter(dsl::id.eq(disk_id)) .set(&DiskUpdate { pantry_address: None }) .check_if_exists::(disk_id) - .execute_and_check(self.pool_authorized(opctx).await?) + .execute_and_check(&*self.pool_connection_authorized(opctx).await?) .await .map(|r| match r.status { UpdateStatus::Updated => true, UpdateStatus::NotUpdatedButExists => false, }) .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_disk), ) @@ -571,7 +571,7 @@ impl DataStore { ok_to_delete_states: &[api::external::DiskState], ) -> Result { use db::schema::disk::dsl; - let pool = self.pool(); + let conn = self.pool_connection_unauthorized().await?; let now = Utc::now(); let ok_to_delete_state_labels: Vec<_> = @@ -585,10 +585,10 @@ impl DataStore { .filter(dsl::attach_instance_id.is_null()) .set((dsl::disk_state.eq(destroyed), dsl::time_deleted.eq(now))) .check_if_exists::(*disk_id) - .execute_and_check(pool) + .execute_and_check(&conn) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::Disk, diff --git a/nexus/db-queries/src/db/datastore/dns.rs b/nexus/db-queries/src/db/datastore/dns.rs index ddf2718930..d9704594b1 100644 --- a/nexus/db-queries/src/db/datastore/dns.rs +++ b/nexus/db-queries/src/db/datastore/dns.rs @@ -6,7 +6,7 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::DnsGroup; use crate::db::model::DnsName; @@ -15,9 +15,10 @@ use crate::db::model::DnsZone; use crate::db::model::Generation; use crate::db::model::InitialDnsGroup; use crate::db::pagination::paginated; +use crate::db::pool::DbConnection; use crate::db::TransactionError; +use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::PoolError; use diesel::prelude::*; use nexus_types::internal_api::params::DnsConfigParams; use nexus_types::internal_api::params::DnsConfigZone; @@ -51,9 +52,9 @@ impl DataStore { paginated(dsl::dns_zone, dsl::zone_name, pagparams) .filter(dsl::dns_group.eq(dns_group)) .select(DnsZone::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// List all DNS zones in a DNS group without pagination @@ -65,25 +66,18 @@ impl DataStore { opctx: &OpContext, dns_group: DnsGroup, ) -> ListResultVec { - let conn = self.pool_authorized(opctx).await?; - self.dns_zones_list_all_on_connection(opctx, conn, dns_group).await + let conn = self.pool_connection_authorized(opctx).await?; + self.dns_zones_list_all_on_connection(opctx, &conn, dns_group).await } /// Variant of [`Self::dns_zones_list_all`] which may be called from a /// transaction context. - pub(crate) async fn dns_zones_list_all_on_connection( + pub(crate) async fn dns_zones_list_all_on_connection( &self, opctx: &OpContext, - conn: &(impl async_bb8_diesel::AsyncConnection< - crate::db::pool::DbConnection, - ConnErr, - > + Sync), + conn: &async_bb8_diesel::Connection, dns_group: DnsGroup, - ) -> ListResultVec - where - ConnErr: From + Send + 'static, - ConnErr: Into, - { + ) -> ListResultVec { use db::schema::dns_zone::dsl; const LIMIT: usize = 5; @@ -95,9 +89,7 @@ impl DataStore { .select(DnsZone::as_select()) .load_async(conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e.into(), ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; bail_unless!( list.len() < LIMIT, @@ -116,25 +108,18 @@ impl DataStore { ) -> LookupResult { self.dns_group_latest_version_conn( opctx, - self.pool_authorized(opctx).await?, + &*self.pool_connection_authorized(opctx).await?, dns_group, ) .await } - pub async fn dns_group_latest_version_conn( + pub async fn dns_group_latest_version_conn( &self, opctx: &OpContext, - conn: &(impl async_bb8_diesel::AsyncConnection< - crate::db::pool::DbConnection, - ConnErr, - > + Sync), + conn: &async_bb8_diesel::Connection, dns_group: DnsGroup, - ) -> LookupResult - where - ConnErr: From + Send + 'static, - ConnErr: Into, - { + ) -> LookupResult { opctx.authorize(authz::Action::Read, &authz::DNS_CONFIG).await?; use db::schema::dns_version::dsl; let versions = dsl::dns_version @@ -144,9 +129,7 @@ impl DataStore { .select(DnsVersion::as_select()) .load_async(conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e.into(), ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; bail_unless!( versions.len() == 1, @@ -178,11 +161,9 @@ impl DataStore { .or(dsl::version_removed.gt(version)), ) .select(DnsName::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })? + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .into_iter() .filter_map(|n: DnsName| match n.records() { Ok(records) => Some((n.name, records)), @@ -326,17 +307,10 @@ impl DataStore { } /// Load initial data for a DNS group into the database - pub async fn load_dns_data( - conn: &(impl async_bb8_diesel::AsyncConnection< - crate::db::pool::DbConnection, - ConnErr, - > + Sync), + pub async fn load_dns_data( + conn: &async_bb8_diesel::Connection, dns: InitialDnsGroup, - ) -> Result<(), Error> - where - ConnErr: From + Send + 'static, - ConnErr: Into, - { + ) -> Result<(), Error> { { use db::schema::dns_zone::dsl; diesel::insert_into(dsl::dns_zone) @@ -346,10 +320,7 @@ impl DataStore { .execute_async(conn) .await .map_err(|e| { - public_error_from_diesel_pool( - e.into(), - ErrorHandler::Server, - ) + public_error_from_diesel(e, ErrorHandler::Server) })?; } @@ -362,10 +333,7 @@ impl DataStore { .execute_async(conn) .await .map_err(|e| { - public_error_from_diesel_pool( - e.into(), - ErrorHandler::Server, - ) + public_error_from_diesel(e, ErrorHandler::Server) })?; } @@ -378,10 +346,7 @@ impl DataStore { .execute_async(conn) .await .map_err(|e| { - public_error_from_diesel_pool( - e.into(), - ErrorHandler::Server, - ) + public_error_from_diesel(e, ErrorHandler::Server) })?; } @@ -407,20 +372,12 @@ impl DataStore { /// **Callers almost certainly want to wake up the corresponding Nexus /// background task to cause these changes to be propagated to the /// corresponding DNS servers.** - pub async fn dns_update( + pub async fn dns_update( &self, opctx: &OpContext, - conn: &(impl async_bb8_diesel::AsyncConnection< - crate::db::pool::DbConnection, - ConnErr, - > + Sync), + conn: &async_bb8_diesel::Connection, update: DnsVersionUpdateBuilder, - ) -> Result<(), Error> - where - ConnErr: From + Send + 'static, - ConnErr: Into, - TransactionError: From, - { + ) -> Result<(), Error> { opctx.authorize(authz::Action::Modify, &authz::DNS_CONFIG).await?; let zones = self @@ -438,28 +395,21 @@ impl DataStore { match result { Ok(()) => Ok(()), Err(TransactionError::CustomError(e)) => Err(e), - Err(TransactionError::Pool(e)) => { - Err(public_error_from_diesel_pool(e, ErrorHandler::Server)) + Err(TransactionError::Connection(e)) => { + Err(public_error_from_diesel(e, ErrorHandler::Server)) } } } // This must only be used inside a transaction. Otherwise, it may make // invalid changes to the database state. Use `dns_update()` instead. - async fn dns_update_internal( + async fn dns_update_internal( &self, opctx: &OpContext, - conn: &(impl async_bb8_diesel::AsyncConnection< - crate::db::pool::DbConnection, - ConnErr, - > + Sync), + conn: &async_bb8_diesel::Connection, update: DnsVersionUpdateBuilder, zones: Vec, - ) -> Result<(), Error> - where - ConnErr: From + Send + 'static, - ConnErr: Into, - { + ) -> Result<(), Error> { // TODO-scalability TODO-performance This would be much better as a CTE // for all the usual reasons described in RFD 192. Using an interactive // transaction here means that either we wind up holding database locks @@ -507,10 +457,7 @@ impl DataStore { .execute_async(conn) .await .map_err(|e| { - public_error_from_diesel_pool( - e.into(), - ErrorHandler::Server, - ) + public_error_from_diesel(e, ErrorHandler::Server) })?; } @@ -534,9 +481,7 @@ impl DataStore { .set(dsl::version_removed.eq(new_version_num)) .execute_async(conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e.into(), ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; bail_unless!( nremoved == ntoremove, @@ -552,10 +497,7 @@ impl DataStore { .execute_async(conn) .await .map_err(|e| { - public_error_from_diesel_pool( - e.into(), - ErrorHandler::Server, - ) + public_error_from_diesel(e, ErrorHandler::Server) })?; bail_unless!( @@ -749,8 +691,8 @@ mod test { comment: "test suite".to_string(), }) .execute_async( - datastore - .pool_for_tests() + &*datastore + .pool_connection_for_tests() .await .expect("failed to get datastore connection"), ) @@ -810,8 +752,8 @@ mod test { HashMap::new(), ); { - let conn = datastore.pool_for_tests().await.unwrap(); - DataStore::load_dns_data(conn, initial) + let conn = datastore.pool_connection_for_tests().await.unwrap(); + DataStore::load_dns_data(&conn, initial) .await .expect("failed to load initial DNS zone"); } @@ -850,8 +792,8 @@ mod test { ]), ); { - let conn = datastore.pool_for_tests().await.unwrap(); - DataStore::load_dns_data(conn, initial) + let conn = datastore.pool_connection_for_tests().await.unwrap(); + DataStore::load_dns_data(&conn, initial) .await .expect("failed to load initial DNS zone"); } @@ -1026,7 +968,9 @@ mod test { zone_name: "z1.foo".to_string(), }, ]) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap(); } @@ -1042,7 +986,9 @@ mod test { vi1.clone(), vi2.clone(), ]) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap(); } @@ -1142,7 +1088,9 @@ mod test { ) .unwrap(), ]) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap(); } @@ -1288,7 +1236,9 @@ mod test { zone_name: "z1.foo".to_string(), }, ]) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap_err(); assert!(error @@ -1317,7 +1267,9 @@ mod test { comment: "test suite 4".to_string(), }, ]) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap_err(); assert!(error @@ -1349,7 +1301,9 @@ mod test { ) .unwrap(), ]) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap_err(); assert!(error @@ -1470,7 +1424,9 @@ mod test { dns_zone2.clone(), dns_zone3.clone(), ]) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap(); } @@ -1494,7 +1450,9 @@ mod test { comment: "test suite 8".to_string(), }, ]) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap(); } @@ -1523,8 +1481,8 @@ mod test { update.add_name(String::from("n1"), records1.clone()).unwrap(); update.add_name(String::from("n2"), records2.clone()).unwrap(); - let conn = datastore.pool_for_tests().await.unwrap(); - datastore.dns_update(&opctx, conn, update).await.unwrap(); + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore.dns_update(&opctx, &conn, update).await.unwrap(); } // Verify the new config. @@ -1556,8 +1514,8 @@ mod test { update.remove_name(String::from("n1")).unwrap(); update.add_name(String::from("n1"), records12.clone()).unwrap(); - let conn = datastore.pool_for_tests().await.unwrap(); - datastore.dns_update(&opctx, conn, update).await.unwrap(); + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore.dns_update(&opctx, &conn, update).await.unwrap(); } let dns_config = datastore @@ -1586,8 +1544,8 @@ mod test { ); update.remove_name(String::from("n1")).unwrap(); - let conn = datastore.pool_for_tests().await.unwrap(); - datastore.dns_update(&opctx, conn, update).await.unwrap(); + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore.dns_update(&opctx, &conn, update).await.unwrap(); } let dns_config = datastore @@ -1613,8 +1571,8 @@ mod test { ); update.add_name(String::from("n1"), records2.clone()).unwrap(); - let conn = datastore.pool_for_tests().await.unwrap(); - datastore.dns_update(&opctx, conn, update).await.unwrap(); + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore.dns_update(&opctx, &conn, update).await.unwrap(); } let dns_config = datastore @@ -1644,8 +1602,8 @@ mod test { ); update1.remove_name(String::from("n1")).unwrap(); - let conn1 = datastore.pool_for_tests().await.unwrap(); - let conn2 = datastore.pool_for_tests().await.unwrap(); + let conn1 = datastore.pool_connection_for_tests().await.unwrap(); + let conn2 = datastore.pool_connection_for_tests().await.unwrap(); let (wait1_tx, wait1_rx) = tokio::sync::oneshot::channel(); let (wait2_tx, wait2_rx) = tokio::sync::oneshot::channel(); @@ -1680,7 +1638,7 @@ mod test { String::from("the test suite"), ); update2.add_name(String::from("n1"), records1.clone()).unwrap(); - datastore.dns_update(&opctx, conn2, update2).await.unwrap(); + datastore.dns_update(&opctx, &conn2, update2).await.unwrap(); // Now let the first one finish. wait2_tx.send(()).unwrap(); @@ -1723,9 +1681,9 @@ mod test { ); update.remove_name(String::from("n4")).unwrap(); - let conn = datastore.pool_for_tests().await.unwrap(); + let conn = datastore.pool_connection_for_tests().await.unwrap(); let error = - datastore.dns_update(&opctx, conn, update).await.unwrap_err(); + datastore.dns_update(&opctx, &conn, update).await.unwrap_err(); assert_eq!( error.to_string(), "Internal Error: updated wrong number of dns_name \ @@ -1748,9 +1706,9 @@ mod test { ); update.add_name(String::from("n2"), records1.clone()).unwrap(); - let conn = datastore.pool_for_tests().await.unwrap(); + let conn = datastore.pool_connection_for_tests().await.unwrap(); let error = - datastore.dns_update(&opctx, conn, update).await.unwrap_err(); + datastore.dns_update(&opctx, &conn, update).await.unwrap_err(); let msg = error.to_string(); assert!(msg.starts_with("Internal Error: ")); assert!(msg.contains("violates unique constraint")); diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs index 8f5e9ba4c1..268b284a0a 100644 --- a/nexus/db-queries/src/db/datastore/external_ip.rs +++ b/nexus/db-queries/src/db/datastore/external_ip.rs @@ -9,7 +9,7 @@ use crate::authz; use crate::authz::ApiResource; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::lookup::LookupPath; use crate::db::model::ExternalIp; @@ -20,7 +20,7 @@ use crate::db::pool::DbConnection; use crate::db::queries::external_ip::NextExternalIp; use crate::db::update_and_check::UpdateAndCheck; use crate::db::update_and_check::UpdateStatus; -use async_bb8_diesel::{AsyncRunQueryDsl, PoolError}; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use nexus_types::identity::Resource; @@ -131,29 +131,22 @@ impl DataStore { opctx: &OpContext, data: IncompleteExternalIp, ) -> CreateResult { - let conn = self.pool_authorized(opctx).await?; - Self::allocate_external_ip_on_connection(conn, data).await + let conn = self.pool_connection_authorized(opctx).await?; + Self::allocate_external_ip_on_connection(&conn, data).await } /// Variant of [Self::allocate_external_ip] which may be called from a /// transaction context. - pub(crate) async fn allocate_external_ip_on_connection( - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + pub(crate) async fn allocate_external_ip_on_connection( + conn: &async_bb8_diesel::Connection, data: IncompleteExternalIp, - ) -> CreateResult - where - ConnErr: From + Send + 'static, - PoolError: From, - { + ) -> CreateResult { let explicit_ip = data.explicit_ip().is_some(); NextExternalIp::new(data).get_result_async(conn).await.map_err(|e| { use async_bb8_diesel::ConnectionError::Query; - use async_bb8_diesel::PoolError::Connection; use diesel::result::Error::NotFound; - let e = PoolError::from(e); match e { - Connection(Query(NotFound)) => { + Query(NotFound) => { if explicit_ip { Error::invalid_request( "Requested external IP address not available", @@ -164,7 +157,7 @@ impl DataStore { ) } } - _ => crate::db::queries::external_ip::from_pool(e), + _ => crate::db::queries::external_ip::from_diesel(e), } }) } @@ -238,13 +231,13 @@ impl DataStore { .filter(dsl::id.eq(ip_id)) .set(dsl::time_deleted.eq(now)) .check_if_exists::(ip_id) - .execute_and_check(self.pool_authorized(opctx).await?) + .execute_and_check(&*self.pool_connection_authorized(opctx).await?) .await .map(|r| match r.status { UpdateStatus::Updated => true, UpdateStatus::NotUpdatedButExists => false, }) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Delete all external IP addresses associated with the provided instance @@ -268,9 +261,9 @@ impl DataStore { .filter(dsl::parent_id.eq(instance_id)) .filter(dsl::kind.ne(IpKind::Floating)) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Fetch all external IP addresses of any kind for the provided instance @@ -285,8 +278,8 @@ impl DataStore { .filter(dsl::parent_id.eq(instance_id)) .filter(dsl::time_deleted.is_null()) .select(ExternalIp::as_select()) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/identity_provider.rs b/nexus/db-queries/src/db/datastore/identity_provider.rs index 4d725d1cf4..fdc9a020e7 100644 --- a/nexus/db-queries/src/db/datastore/identity_provider.rs +++ b/nexus/db-queries/src/db/datastore/identity_provider.rs @@ -8,7 +8,7 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::identity::Resource; use crate::db::model::IdentityProvider; @@ -46,9 +46,11 @@ impl DataStore { .filter(dsl::silo_id.eq(authz_idp_list.silo().id())) .filter(dsl::time_deleted.is_null()) .select(IdentityProvider::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn saml_identity_provider_create( @@ -61,7 +63,7 @@ impl DataStore { assert_eq!(provider.silo_id, authz_idp_list.silo().id()); let name = provider.identity().name.to_string(); - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { // insert silo identity provider record with type Saml @@ -94,7 +96,7 @@ impl DataStore { }) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SamlIdentityProvider, diff --git a/nexus/db-queries/src/db/datastore/image.rs b/nexus/db-queries/src/db/datastore/image.rs index 17bdb6fae0..e44da013cd 100644 --- a/nexus/db-queries/src/db/datastore/image.rs +++ b/nexus/db-queries/src/db/datastore/image.rs @@ -4,7 +4,7 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::Image; use crate::db::model::Project; @@ -52,9 +52,11 @@ impl DataStore { .filter(project_dsl::time_deleted.is_null()) .filter(project_dsl::project_id.eq(authz_project.id())) .select(ProjectImage::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) .map(|v| v.into_iter().map(|v| v.into()).collect()) } @@ -80,9 +82,11 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::silo_id.eq(authz_silo.id())) .select(SiloImage::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) .map(|v| v.into_iter().map(|v| v.into()).collect()) } @@ -107,19 +111,19 @@ impl DataStore { .do_update() .set(dsl::time_modified.eq(dsl::time_modified)), ) - .insert_and_get_result_async(self.pool_authorized(opctx).await?) + .insert_and_get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => authz_silo.not_found(), - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::ProjectImage, - name.as_str(), - ), - ) - } + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::ProjectImage, + name.as_str(), + ), + ), })?; Ok(image) } @@ -145,19 +149,19 @@ impl DataStore { .do_update() .set(dsl::time_modified.eq(dsl::time_modified)), ) - .insert_and_get_result_async(self.pool_authorized(opctx).await?) + .insert_and_get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => authz_project.not_found(), - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::ProjectImage, - name.as_str(), - ), - ) - } + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::ProjectImage, + name.as_str(), + ), + ), })?; Ok(image) } @@ -181,10 +185,10 @@ impl DataStore { dsl::time_modified.eq(Utc::now()), )) .returning(Image::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SiloImage, @@ -215,10 +219,10 @@ impl DataStore { dsl::time_modified.eq(Utc::now()), )) .returning(Image::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::ProjectImage, diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index 1f347d2378..46ca07a74a 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -13,7 +13,7 @@ use crate::db::collection_detach_many::DatastoreDetachManyTarget; use crate::db::collection_detach_many::DetachManyError; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::identity::Resource; use crate::db::lookup::LookupPath; @@ -84,19 +84,16 @@ impl DataStore { .do_update() .set(dsl::time_modified.eq(dsl::time_modified)), ) - .insert_and_get_result_async(self.pool_authorized(opctx).await?) + .insert_and_get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => authz_project.not_found(), - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::Instance, - name.as_str(), - ), - ) - } + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict(ResourceType::Instance, name.as_str()), + ), })?; bail_unless!( @@ -135,9 +132,9 @@ impl DataStore { .filter(dsl::project_id.eq(authz_project.id())) .filter(dsl::time_deleted.is_null()) .select(Instance::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Fetches information about an Instance that the caller has previously @@ -178,10 +175,10 @@ impl DataStore { .filter(dsl::id.eq(authz_instance.id())) .filter(dsl::time_deleted.is_not_null()) .select(Instance::as_select()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, @@ -225,14 +222,14 @@ impl DataStore { ) .set(new_runtime.clone()) .check_if_exists::(*instance_id) - .execute_and_check(self.pool()) + .execute_and_check(&*self.pool_connection_unauthorized().await?) .await .map(|r| match r.status { UpdateStatus::Updated => true, UpdateStatus::NotUpdatedButExists => false, }) .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, @@ -288,7 +285,9 @@ impl DataStore { disk::dsl::slot.eq(Option::::None), )), ) - .detach_and_get_result_async(self.pool_authorized(opctx).await?) + .detach_and_get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) .await .map_err(|e| match e { DetachManyError::CollectionNotFound => Error::not_found_by_id( @@ -309,7 +308,7 @@ impl DataStore { } } DetachManyError::DatabaseError(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) } })?; diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs index 1248edf7a8..bd3148f2f7 100644 --- a/nexus/db-queries/src/db/datastore/ip_pool.rs +++ b/nexus/db-queries/src/db/datastore/ip_pool.rs @@ -10,8 +10,8 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::diesel_pool_result_optional; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::diesel_result_optional; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::fixed_data::silo::INTERNAL_SILO_ID; use crate::db::identity::Resource; @@ -22,7 +22,7 @@ use crate::db::model::Name; use crate::db::pagination::paginated; use crate::db::pool::DbConnection; use crate::db::queries::ip_pool::FilterOverlappingIpRanges; -use async_bb8_diesel::{AsyncRunQueryDsl, PoolError}; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use ipnetwork::IpNetwork; @@ -65,9 +65,9 @@ impl DataStore { .filter(dsl::silo_id.ne(*INTERNAL_SILO_ID).or(dsl::silo_id.is_null())) .filter(dsl::time_deleted.is_null()) .select(db::model::IpPool::as_select()) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Look up the default IP pool for the current silo. If there is no default @@ -104,9 +104,11 @@ impl DataStore { // then by only taking the first result, we get the most specific one .order(dsl::silo_id.asc().nulls_last()) .select(IpPool::as_select()) - .first_async::(self.pool_authorized(opctx).await?) + .first_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Looks up an IP pool intended for internal services. @@ -127,9 +129,9 @@ impl DataStore { .filter(dsl::silo_id.eq(*INTERNAL_SILO_ID)) .filter(dsl::time_deleted.is_null()) .select(IpPool::as_select()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) .map(|ip_pool| { ( authz::IpPool::new( @@ -160,10 +162,10 @@ impl DataStore { diesel::insert_into(dsl::ip_pool) .values(pool) .returning(IpPool::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict(ResourceType::IpPool, &pool_name), ) @@ -181,16 +183,18 @@ impl DataStore { opctx.authorize(authz::Action::Delete, authz_pool).await?; // Verify there are no IP ranges still in this pool - let range = diesel_pool_result_optional( + let range = diesel_result_optional( ip_pool_range::dsl::ip_pool_range .filter(ip_pool_range::dsl::ip_pool_id.eq(authz_pool.id())) .filter(ip_pool_range::dsl::time_deleted.is_null()) .select(ip_pool_range::dsl::id) .limit(1) - .first_async::(self.pool_authorized(opctx).await?) + .first_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await, ) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if range.is_some() { return Err(Error::InvalidRequest { message: @@ -212,10 +216,10 @@ impl DataStore { .filter(dsl::id.eq(authz_pool.id())) .filter(dsl::rcgen.eq(db_pool.rcgen)) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_pool), ) @@ -247,10 +251,10 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .set(updates) .returning(IpPool::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_pool), ) @@ -269,10 +273,10 @@ impl DataStore { .filter(dsl::ip_pool_id.eq(authz_pool.id())) .filter(dsl::time_deleted.is_null()) .select(IpPoolRange::as_select()) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_pool), ) @@ -285,24 +289,19 @@ impl DataStore { authz_pool: &authz::IpPool, range: &IpRange, ) -> CreateResult { - let conn = self.pool_authorized(opctx).await?; - Self::ip_pool_add_range_on_connection(conn, opctx, authz_pool, range) + let conn = self.pool_connection_authorized(opctx).await?; + Self::ip_pool_add_range_on_connection(&conn, opctx, authz_pool, range) .await } /// Variant of [Self::ip_pool_add_range] which may be called from a /// transaction context. - pub(crate) async fn ip_pool_add_range_on_connection( - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + pub(crate) async fn ip_pool_add_range_on_connection( + conn: &async_bb8_diesel::Connection, opctx: &OpContext, authz_pool: &authz::IpPool, range: &IpRange, - ) -> CreateResult - where - ConnErr: From + Send + 'static, - PoolError: From, - { + ) -> CreateResult { use db::schema::ip_pool_range::dsl; opctx.authorize(authz::Action::CreateChild, authz_pool).await?; let pool_id = authz_pool.id(); @@ -315,13 +314,16 @@ impl DataStore { .await .map_err(|e| { use async_bb8_diesel::ConnectionError::Query; - use async_bb8_diesel::PoolError::Connection; use diesel::result::Error::NotFound; match e { - AsyncInsertError::DatabaseError(Connection(Query( - NotFound, - ))) => { + AsyncInsertError::CollectionNotFound => { + Error::ObjectNotFound { + type_name: ResourceType::IpPool, + lookup_type: LookupType::ById(pool_id), + } + } + AsyncInsertError::DatabaseError(Query(NotFound)) => { // We've filtered out the IP addresses the client provided, // i.e., there's some overlap with existing addresses. Error::invalid_request( @@ -334,14 +336,8 @@ impl DataStore { .as_str(), ) } - AsyncInsertError::CollectionNotFound => { - Error::ObjectNotFound { - type_name: ResourceType::IpPool, - lookup_type: LookupType::ById(pool_id), - } - } AsyncInsertError::DatabaseError(err) => { - public_error_from_diesel_pool(err, ErrorHandler::Server) + public_error_from_diesel(err, ErrorHandler::Server) } } }) @@ -366,19 +362,18 @@ impl DataStore { // Fetch the range itself, if it exists. We'll need to protect against // concurrent inserts of new external IPs from the target range by // comparing the rcgen. - let range = diesel_pool_result_optional( + let conn = self.pool_connection_authorized(opctx).await?; + let range = diesel_result_optional( dsl::ip_pool_range .filter(dsl::ip_pool_id.eq(pool_id)) .filter(dsl::first_address.eq(first_net)) .filter(dsl::last_address.eq(last_net)) .filter(dsl::time_deleted.is_null()) .select(IpPoolRange::as_select()) - .get_result_async::( - self.pool_authorized(opctx).await?, - ) + .get_result_async::(&*conn) .await, ) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))? + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .ok_or_else(|| { Error::invalid_request( format!( @@ -397,9 +392,9 @@ impl DataStore { .filter(external_ip::dsl::ip_pool_range_id.eq(range_id)) .filter(external_ip::dsl::time_deleted.is_null()), )) - .get_result_async::(self.pool_authorized(opctx).await?) + .get_result_async::(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if has_children { return Err(Error::invalid_request( "IP pool ranges cannot be deleted while \ @@ -419,9 +414,9 @@ impl DataStore { .filter(dsl::rcgen.eq(rcgen)), ) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if updated_rows == 1 { Ok(()) } else { diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index f653675728..ff1df710bb 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -24,7 +24,7 @@ use crate::authz; use crate::context::OpContext; use crate::db::{ self, - error::{public_error_from_diesel_pool, ErrorHandler}, + error::{public_error_from_diesel, ErrorHandler}, }; use ::oximeter::types::ProducerRegistry; use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionManager}; @@ -200,16 +200,7 @@ impl DataStore { .unwrap(); } - // TODO-security This should be deprecated in favor of pool_authorized(), - // which gives us the chance to do a minimal security check before hitting - // the database. Eventually, this function should only be used for doing - // authentication in the first place (since we can't do an authz check in - // that case). - fn pool(&self) -> &bb8::Pool> { - self.pool.pool() - } - - pub(super) async fn pool_authorized( + async fn pool_authorized( &self, opctx: &OpContext, ) -> Result<&bb8::Pool>, Error> { @@ -217,12 +208,41 @@ impl DataStore { Ok(self.pool.pool()) } + /// Returns a connection to a connection from the database connection pool. + pub(super) async fn pool_connection_authorized( + &self, + opctx: &OpContext, + ) -> Result>, Error> + { + let pool = self.pool_authorized(opctx).await?; + let connection = pool.get().await.map_err(|err| { + Error::unavail(&format!("Failed to access DB connection: {err}")) + })?; + Ok(connection) + } + + /// Returns an unauthorized connection to a connection from the database + /// connection pool. + /// + /// TODO-security: This should be deprecated in favor of + /// "pool_connection_authorized". + pub(super) async fn pool_connection_unauthorized( + &self, + ) -> Result>, Error> + { + let connection = self.pool.pool().get().await.map_err(|err| { + Error::unavail(&format!("Failed to access DB connection: {err}")) + })?; + Ok(connection) + } + /// For testing only. This isn't cfg(test) because nexus needs access to it. #[doc(hidden)] - pub async fn pool_for_tests( + pub async fn pool_connection_for_tests( &self, - ) -> Result<&bb8::Pool>, Error> { - Ok(self.pool.pool()) + ) -> Result>, Error> + { + self.pool_connection_unauthorized().await } /// Return the next available IPv6 address for an Oxide service running on @@ -238,10 +258,10 @@ impl DataStore { ) .set(dsl::last_used_address.eq(dsl::last_used_address + 1)) .returning(dsl::last_used_address) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::Sled, @@ -266,19 +286,17 @@ impl DataStore { #[cfg(test)] async fn test_try_table_scan(&self, opctx: &OpContext) -> Error { use db::schema::project::dsl; - let conn = self.pool_authorized(opctx).await; + let conn = self.pool_connection_authorized(opctx).await; if let Err(error) = conn { return error; } let result = dsl::project .select(diesel::dsl::count_star()) - .first_async::(conn.unwrap()) + .first_async::(&*conn.unwrap()) .await; match result { Ok(_) => Error::internal_error("table scan unexpectedly succeeded"), - Err(error) => { - public_error_from_diesel_pool(error, ErrorHandler::Server) - } + Err(error) => public_error_from_diesel(error, ErrorHandler::Server), } } } @@ -1000,9 +1018,9 @@ mod test { let pool = db::Pool::new(&logctx.log, &cfg); let datastore = DataStore::new(&logctx.log, Arc::new(pool), None).await.unwrap(); - + let conn = datastore.pool_connection_for_tests().await.unwrap(); let explanation = DataStore::get_allocated_regions_query(Uuid::nil()) - .explain_async(datastore.pool()) + .explain_async(&conn) .await .unwrap(); assert!( @@ -1027,7 +1045,7 @@ mod test { .values(values) .returning(VpcSubnet::as_returning()); println!("{}", diesel::debug_query(&query)); - let explanation = query.explain_async(datastore.pool()).await.unwrap(); + let explanation = query.explain_async(&conn).await.unwrap(); assert!( !explanation.contains("FULL SCAN"), "Found an unexpected FULL SCAN: {}", @@ -1403,6 +1421,7 @@ mod test { ); let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; + let conn = datastore.pool_connection_for_tests().await.unwrap(); // Create a few records. let now = Utc::now(); @@ -1429,7 +1448,7 @@ mod test { .collect::>(); diesel::insert_into(dsl::external_ip) .values(ips.clone()) - .execute_async(datastore.pool()) + .execute_async(&*conn) .await .unwrap(); @@ -1464,6 +1483,7 @@ mod test { dev::test_setup_log("test_deallocate_external_ip_is_idempotent"); let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; + let conn = datastore.pool_connection_for_tests().await.unwrap(); // Create a record. let now = Utc::now(); @@ -1487,7 +1507,7 @@ mod test { }; diesel::insert_into(dsl::external_ip) .values(ip.clone()) - .execute_async(datastore.pool()) + .execute_async(&*conn) .await .unwrap(); @@ -1522,13 +1542,13 @@ mod test { use crate::db::model::IpKind; use crate::db::schema::external_ip::dsl; use async_bb8_diesel::ConnectionError::Query; - use async_bb8_diesel::PoolError::Connection; use diesel::result::DatabaseErrorKind::CheckViolation; use diesel::result::Error::DatabaseError; let logctx = dev::test_setup_log("test_external_ip_check_constraints"); let mut db = test_setup_database(&logctx.log).await; let (_opctx, datastore) = datastore_test(&logctx, &db).await; + let conn = datastore.pool_connection_for_tests().await.unwrap(); let now = Utc::now(); // Create a mostly-populated record, for a floating IP @@ -1582,7 +1602,7 @@ mod test { }; let res = diesel::insert_into(dsl::external_ip) .values(new_ip) - .execute_async(datastore.pool()) + .execute_async(&*conn) .await; if name.is_some() && description.is_some() { // Name/description must be non-NULL, instance ID can be @@ -1607,10 +1627,10 @@ mod test { assert!( matches!( err, - Connection(Query(DatabaseError( + Query(DatabaseError( CheckViolation, _ - ))) + )) ), "Expected a CHECK violation when inserting a \ Floating IP record with NULL name and/or description", @@ -1639,7 +1659,7 @@ mod test { }; let res = diesel::insert_into(dsl::external_ip) .values(new_ip.clone()) - .execute_async(datastore.pool()) + .execute_async(&*conn) .await; let ip_type = if is_service { "Service" } else { "Instance" }; @@ -1656,10 +1676,10 @@ mod test { assert!( matches!( err, - Connection(Query(DatabaseError( + Query(DatabaseError( CheckViolation, _ - ))) + )) ), "Expected a CHECK violation when inserting an \ Ephemeral Service IP", @@ -1687,10 +1707,10 @@ mod test { assert!( matches!( err, - Connection(Query(DatabaseError( + Query(DatabaseError( CheckViolation, _ - ))) + )) ), "Expected a CHECK violation when inserting a \ {:?} IP record with non-NULL name, description, \ diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index af1068d6bf..3d7b8afa71 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -11,7 +11,7 @@ use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::cte_utils::BoxedQuery; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::model::IncompleteNetworkInterface; @@ -27,7 +27,6 @@ use crate::db::pool::DbConnection; use crate::db::queries::network_interface; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::PoolError; use chrono::Utc; use diesel::prelude::*; use omicron_common::api::external; @@ -156,21 +155,17 @@ impl DataStore { interface: IncompleteNetworkInterface, ) -> Result { let conn = self - .pool_authorized(opctx) + .pool_connection_authorized(opctx) .await .map_err(network_interface::InsertError::External)?; - self.create_network_interface_raw_conn(conn, interface).await + self.create_network_interface_raw_conn(&conn, interface).await } - pub(crate) async fn create_network_interface_raw_conn( + pub(crate) async fn create_network_interface_raw_conn( &self, - conn: &(impl AsyncConnection + Sync), + conn: &async_bb8_diesel::Connection, interface: IncompleteNetworkInterface, - ) -> Result - where - ConnErr: From + Send + 'static, - PoolError: From, - { + ) -> Result { use db::schema::network_interface::dsl; let subnet_id = interface.subnet.identity.id; let query = network_interface::InsertQuery::new(interface.clone()); @@ -190,7 +185,7 @@ impl DataStore { ) } AsyncInsertError::DatabaseError(e) => { - network_interface::InsertError::from_pool(e, &interface) + network_interface::InsertError::from_diesel(e, &interface) } }) } @@ -210,10 +205,10 @@ impl DataStore { .filter(dsl::kind.eq(NetworkInterfaceKind::Instance)) .filter(dsl::time_deleted.is_null()) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_instance), ) @@ -243,13 +238,14 @@ impl DataStore { query .clone() .execute_async( - self.pool_authorized(opctx) + &*self + .pool_connection_authorized(opctx) .await .map_err(network_interface::DeleteError::External)?, ) .await .map_err(|e| { - network_interface::DeleteError::from_pool(e, &query) + network_interface::DeleteError::from_diesel(e, &query) })?; Ok(()) } @@ -291,11 +287,11 @@ impl DataStore { network_interface::is_primary, network_interface::slot, )) - .get_results_async::(self.pool_authorized(opctx).await?) + .get_results_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; Ok(rows .into_iter() .map(sled_client_types::NetworkInterface::from) @@ -386,10 +382,10 @@ impl DataStore { .filter(dsl::instance_id.eq(authz_instance.id())) .select(InstanceNetworkInterface::as_select()) .load_async::( - self.pool_authorized(opctx).await?, + &*self.pool_connection_authorized(opctx).await?, ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Update a network interface associated with a given instance. @@ -471,9 +467,9 @@ impl DataStore { } type TxnError = TransactionError; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; if primary { - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { let instance_state = instance_query .get_result_async(&conn) .await? @@ -517,7 +513,7 @@ impl DataStore { // be done there. The other columns always need to be updated, and // we're only hitting a single row. Note that we still need to // verify the instance is stopped. - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { let instance_state = instance_query .get_result_async(&conn) .await? diff --git a/nexus/db-queries/src/db/datastore/oximeter.rs b/nexus/db-queries/src/db/datastore/oximeter.rs index 178c2466a7..c9b3a59b05 100644 --- a/nexus/db-queries/src/db/datastore/oximeter.rs +++ b/nexus/db-queries/src/db/datastore/oximeter.rs @@ -6,7 +6,7 @@ use super::DataStore; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::OximeterInfo; use crate::db::model::ProducerEndpoint; @@ -41,10 +41,10 @@ impl DataStore { dsl::ip.eq(info.ip), dsl::port.eq(info.port), )) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_unauthorized().await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::Oximeter, @@ -62,9 +62,11 @@ impl DataStore { ) -> ListResultVec { use db::schema::oximeter::dsl; paginated(dsl::oximeter, dsl::id, page_params) - .load_async::(self.pool()) + .load_async::( + &*self.pool_connection_unauthorized().await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } // Create a record for a new producer endpoint @@ -86,10 +88,10 @@ impl DataStore { dsl::interval.eq(producer.interval), dsl::base_route.eq(producer.base_route.clone()), )) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_unauthorized().await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::MetricProducer, @@ -111,10 +113,10 @@ impl DataStore { .filter(dsl::oximeter_id.eq(oximeter_id)) .order_by((dsl::oximeter_id, dsl::id)) .select(ProducerEndpoint::as_select()) - .load_async(self.pool()) + .load_async(&*self.pool_connection_unauthorized().await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::MetricProducer, diff --git a/nexus/db-queries/src/db/datastore/physical_disk.rs b/nexus/db-queries/src/db/datastore/physical_disk.rs index ec9f29d27d..3c83b91d21 100644 --- a/nexus/db-queries/src/db/datastore/physical_disk.rs +++ b/nexus/db-queries/src/db/datastore/physical_disk.rs @@ -10,7 +10,7 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::PhysicalDisk; use crate::db::model::Sled; @@ -60,7 +60,9 @@ impl DataStore { dsl::time_modified.eq(now), )), ) - .insert_and_get_result_async(self.pool()) + .insert_and_get_result_async( + &*self.pool_connection_authorized(&opctx).await?, + ) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { @@ -68,7 +70,7 @@ impl DataStore { lookup_type: LookupType::ById(sled_id), }, AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) } })?; @@ -85,9 +87,9 @@ impl DataStore { paginated(dsl::physical_disk, dsl::id, pagparams) .filter(dsl::time_deleted.is_null()) .select(PhysicalDisk::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn sled_list_physical_disks( @@ -102,9 +104,9 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::sled_id.eq(sled_id)) .select(PhysicalDisk::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Deletes a disk from the database. @@ -125,10 +127,10 @@ impl DataStore { .filter(dsl::model.eq(model)) .filter(dsl::sled_id.eq(sled_id)) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map(|_rows_modified| ()) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/project.rs b/nexus/db-queries/src/db/datastore/project.rs index b3759f9cce..0285679cd5 100644 --- a/nexus/db-queries/src/db/datastore/project.rs +++ b/nexus/db-queries/src/db/datastore/project.rs @@ -11,8 +11,8 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::diesel_pool_result_optional; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::diesel_result_optional; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::fixed_data::project::SERVICES_PROJECT; @@ -25,7 +25,7 @@ use crate::db::model::ProjectUpdate; use crate::db::model::Silo; use crate::db::model::VirtualProvisioningCollection; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, PoolError}; +use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; use chrono::Utc; use diesel::prelude::*; use omicron_common::api::external::http_pagination::PaginatedBy; @@ -60,16 +60,16 @@ macro_rules! generate_fn_to_ensure_none_in_project { ) -> DeleteResult { use db::schema::$i; - let maybe_label = diesel_pool_result_optional( + let maybe_label = diesel_result_optional( $i::dsl::$i .filter($i::dsl::project_id.eq(authz_project.id())) .filter($i::dsl::time_deleted.is_null()) .select($i::dsl::$label) .limit(1) - .first_async::<$label_ty>(self.pool_authorized(opctx).await?) + .first_async::<$label_ty>(&*self.pool_connection_authorized(opctx).await?) .await, ) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if let Some(label) = maybe_label { let object = stringify!($i).replace('_', " "); @@ -155,7 +155,7 @@ impl DataStore { let name = project.name().as_str().to_string(); let db_project = self - .pool_authorized(opctx) + .pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { let project: Project = Silo::insert_resource( @@ -169,7 +169,7 @@ impl DataStore { authz_silo_inner.not_found() } AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::Project, @@ -193,8 +193,8 @@ impl DataStore { .await .map_err(|e| match e { TransactionError::CustomError(e) => e, - TransactionError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TransactionError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } })?; @@ -233,7 +233,7 @@ impl DataStore { use db::schema::project::dsl; type TxnError = TransactionError; - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { let now = Utc::now(); @@ -246,8 +246,8 @@ impl DataStore { .execute_async(&conn) .await .map_err(|e| { - public_error_from_diesel_pool( - PoolError::from(e), + public_error_from_diesel( + e, ErrorHandler::NotFoundByResource(authz_project), ) })?; @@ -270,8 +270,8 @@ impl DataStore { .await .map_err(|e| match e { TxnError::CustomError(e) => e, - TxnError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TxnError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } })?; Ok(()) @@ -300,9 +300,9 @@ impl DataStore { .filter(dsl::silo_id.eq(authz_silo.id())) .filter(dsl::time_deleted.is_null()) .select(Project::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Updates a project (clobbering update -- no etag) @@ -320,10 +320,10 @@ impl DataStore { .filter(dsl::id.eq(authz_project.id())) .set(updates) .returning(Project::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_project), ) @@ -355,8 +355,8 @@ impl DataStore { .filter(dsl::silo_id.ne(*INTERNAL_SILO_ID).or(dsl::silo_id.is_null())) .filter(dsl::time_deleted.is_null()) .select(db::model::IpPool::as_select()) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 54346b31c0..1be3e1ee4c 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -12,7 +12,7 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::fixed_data::silo::INTERNAL_SILO_ID; @@ -28,7 +28,6 @@ use crate::db::pagination::paginated; use crate::db::pool::DbConnection; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::PoolError; use chrono::Utc; use diesel::prelude::*; use diesel::upsert::excluded; @@ -80,7 +79,7 @@ enum RackInitError { AddingNic(Error), ServiceInsert(Error), DatasetInsert { err: AsyncInsertError, zpool_id: Uuid }, - RackUpdate { err: PoolError, rack_id: Uuid }, + RackUpdate { err: async_bb8_diesel::ConnectionError, rack_id: Uuid }, DnsSerialization(Error), Silo(Error), RoleAssignment(Error), @@ -101,7 +100,7 @@ impl From for Error { lookup_type: LookupType::ById(zpool_id), }, AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) } }, TxnError::CustomError(RackInitError::ServiceInsert(err)) => { @@ -113,7 +112,7 @@ impl From for Error { TxnError::CustomError(RackInitError::RackUpdate { err, rack_id, - }) => public_error_from_diesel_pool( + }) => public_error_from_diesel( err, ErrorHandler::NotFoundByLookup( ResourceType::Rack, @@ -138,7 +137,7 @@ impl From for Error { err )) } - TxnError::Pool(e) => { + TxnError::Connection(e) => { Error::internal_error(&format!("Transaction error: {}", e)) } } @@ -155,9 +154,9 @@ impl DataStore { use db::schema::rack::dsl; paginated(dsl::rack, dsl::id, pagparams) .select(Rack::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Stores a new rack in the database. @@ -177,10 +176,10 @@ impl DataStore { // This is a no-op, since we conflicted on the ID. .set(dsl::id.eq(excluded(dsl::id))) .returning(Rack::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::Rack, @@ -194,25 +193,17 @@ impl DataStore { // which comes from the transaction created in `rack_set_initialized`. #[allow(clippy::too_many_arguments)] - async fn rack_create_recovery_silo( + async fn rack_create_recovery_silo( &self, opctx: &OpContext, - conn: &(impl AsyncConnection + Sync), + conn: &async_bb8_diesel::Connection, log: &slog::Logger, recovery_silo: external_params::SiloCreate, recovery_silo_fq_dns_name: String, recovery_user_id: external_params::UserId, recovery_user_password_hash: omicron_passwords::PasswordHashString, dns_update: DnsVersionUpdateBuilder, - ) -> Result<(), TxnError> - where - ConnError: From + Send + 'static, - PoolError: From, - TransactionError: From, - TxnError: From, - async_bb8_diesel::Connection: - AsyncConnection, - { + ) -> Result<(), TxnError> { let db_silo = self .silo_create_conn( conn, @@ -289,17 +280,13 @@ impl DataStore { Ok(()) } - async fn rack_populate_service_records( + async fn rack_populate_service_records( &self, - conn: &(impl AsyncConnection + Sync), + conn: &async_bb8_diesel::Connection, log: &slog::Logger, service_pool: &db::model::IpPool, service: internal_params::ServicePutRequest, - ) -> Result<(), TxnError> - where - ConnError: From + Send + 'static, - PoolError: From, - { + ) -> Result<(), TxnError> { use internal_params::ServiceKind; let service_db = db::model::Service::new( @@ -431,7 +418,7 @@ impl DataStore { // the low-frequency of calls, this optimization has been deferred. let log = opctx.log.clone(); let rack = self - .pool_authorized(opctx) + .pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { // Early exit if the rack has already been initialized. @@ -443,7 +430,7 @@ impl DataStore { .map_err(|e| { warn!(log, "Initializing Rack: Rack UUID not found"); TxnError::CustomError(RackInitError::RackUpdate { - err: PoolError::from(e), + err: e, rack_id, }) })?; @@ -548,9 +535,9 @@ impl DataStore { .returning(Rack::as_returning()) .get_result_async::(&conn) .await - .map_err(|e| { + .map_err(|err| { TxnError::CustomError(RackInitError::RackUpdate { - err: PoolError::from(e), + err, rack_id, }) })?; @@ -612,7 +599,7 @@ impl DataStore { use crate::db::schema::external_ip::dsl as extip_dsl; use crate::db::schema::service::dsl as service_dsl; type TxnError = TransactionError; - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { let ips = extip_dsl::external_ip @@ -644,8 +631,8 @@ impl DataStore { .await .map_err(|error: TxnError| match error { TransactionError::CustomError(err) => err, - TransactionError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TransactionError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } }) } @@ -879,7 +866,7 @@ mod test { async fn [](db: &DataStore) -> Vec<$model> { use crate::db::schema::$table::dsl; use nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL; - db.pool_for_tests() + db.pool_connection_for_tests() .await .unwrap() .transaction_async(|conn| async move { diff --git a/nexus/db-queries/src/db/datastore/region.rs b/nexus/db-queries/src/db/datastore/region.rs index 6bfea9085d..5bc79b9481 100644 --- a/nexus/db-queries/src/db/datastore/region.rs +++ b/nexus/db-queries/src/db/datastore/region.rs @@ -9,7 +9,7 @@ use super::RegionAllocationStrategy; use super::RunnableQuery; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::lookup::LookupPath; @@ -51,9 +51,11 @@ impl DataStore { volume_id: Uuid, ) -> Result, Error> { Self::get_allocated_regions_query(volume_id) - .get_results_async::<(Dataset, Region)>(self.pool()) + .get_results_async::<(Dataset, Region)>( + &*self.pool_connection_unauthorized().await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } async fn get_block_size_from_disk_source( @@ -136,9 +138,11 @@ impl DataStore { extent_count, allocation_strategy, ) - .get_results_async(self.pool()) + .get_results_async(&*self.pool_connection_authorized(&opctx).await?) .await - .map_err(|e| crate::db::queries::region_allocation::from_pool(e))?; + .map_err(|e| { + crate::db::queries::region_allocation::from_diesel(e) + })?; Ok(dataset_and_regions) } @@ -168,8 +172,9 @@ impl DataStore { // transaction" error. let transaction = { |region_ids: Vec| async { - self.pool() - .transaction(move |conn| { + self.pool_connection_unauthorized() + .await? + .transaction_async(|conn| async move { use db::schema::dataset::dsl as dataset_dsl; use db::schema::region::dsl as region_dsl; @@ -177,7 +182,7 @@ impl DataStore { let datasets = diesel::delete(region_dsl::region) .filter(region_dsl::id.eq_any(region_ids)) .returning(region_dsl::dataset_id) - .get_results::(conn)?; + .get_results_async::(&conn).await?; // Update datasets to which the regions belonged. for dataset in datasets { @@ -191,7 +196,7 @@ impl DataStore { * region_dsl::extent_count, )) .nullable() - .get_result(conn)?; + .get_result_async(&conn).await?; let dataset_total_occupied_size: i64 = if let Some( dataset_total_occupied_size, @@ -220,7 +225,7 @@ impl DataStore { dataset_dsl::size_used .eq(dataset_total_occupied_size), ) - .execute(conn)?; + .execute_async(&conn).await?; } Ok(()) @@ -269,10 +274,10 @@ impl DataStore { * region_dsl::extent_count, )) .nullable() - .get_result_async(self.pool()) + .get_result_async(&*self.pool_connection_unauthorized().await?) .await .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) })?; if let Some(total_occupied_size) = total_occupied_size { diff --git a/nexus/db-queries/src/db/datastore/region_snapshot.rs b/nexus/db-queries/src/db/datastore/region_snapshot.rs index dab3a90bcb..0a707e4504 100644 --- a/nexus/db-queries/src/db/datastore/region_snapshot.rs +++ b/nexus/db-queries/src/db/datastore/region_snapshot.rs @@ -6,7 +6,7 @@ use super::DataStore; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::RegionSnapshot; use async_bb8_diesel::AsyncRunQueryDsl; @@ -25,10 +25,10 @@ impl DataStore { diesel::insert_into(dsl::region_snapshot) .values(region_snapshot.clone()) .on_conflict_do_nothing() - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_unauthorized().await?) .await .map(|_| ()) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn region_snapshot_remove( @@ -43,9 +43,9 @@ impl DataStore { .filter(dsl::dataset_id.eq(dataset_id)) .filter(dsl::region_id.eq(region_id)) .filter(dsl::snapshot_id.eq(snapshot_id)) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_unauthorized().await?) .await .map(|_rows_deleted| ()) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/role.rs b/nexus/db-queries/src/db/datastore/role.rs index ba217ff350..f1198c239b 100644 --- a/nexus/db-queries/src/db/datastore/role.rs +++ b/nexus/db-queries/src/db/datastore/role.rs @@ -11,7 +11,7 @@ use crate::context::OpContext; use crate::db; use crate::db::datastore::RunnableQuery; use crate::db::datastore::RunnableQueryNoReturn; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::fixed_data::role_assignment::BUILTIN_ROLE_ASSIGNMENTS; @@ -24,7 +24,6 @@ use crate::db::pagination::paginated_multicolumn; use crate::db::pool::DbConnection; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::PoolError; use diesel::prelude::*; use nexus_types::external_api::shared; use omicron_common::api::external::DataPageParams; @@ -43,15 +42,17 @@ impl DataStore { ) -> ListResultVec { use db::schema::role_builtin::dsl; opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + + let conn = self.pool_connection_authorized(opctx).await?; paginated_multicolumn( dsl::role_builtin, (dsl::resource_type, dsl::role_name), pagparams, ) .select(RoleBuiltin::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Load built-in roles into the database @@ -75,15 +76,14 @@ impl DataStore { .collect::>(); debug!(opctx.log, "attempting to create built-in roles"); + let conn = self.pool_connection_authorized(opctx).await?; let count = diesel::insert_into(dsl::role_builtin) .values(builtin_roles) .on_conflict((dsl::resource_type, dsl::role_name)) .do_nothing() - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; info!(opctx.log, "created {} built-in roles", count); Ok(()) } @@ -99,6 +99,7 @@ impl DataStore { opctx.authorize(authz::Action::Modify, &authz::DATABASE).await?; debug!(opctx.log, "attempting to create built-in role assignments"); + let conn = self.pool_connection_authorized(opctx).await?; let count = diesel::insert_into(dsl::role_assignment) .values(&*BUILTIN_ROLE_ASSIGNMENTS) .on_conflict(( @@ -109,11 +110,9 @@ impl DataStore { dsl::role_name, )) .do_nothing() - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; info!(opctx.log, "created {} built-in role assignments", count); Ok(()) } @@ -141,7 +140,7 @@ impl DataStore { // into some hurt by assigning loads of roles to someone and having that // person attempt to access anything. - self.pool_authorized(opctx).await? + self.pool_connection_authorized(opctx).await? .transaction_async(|conn| async move { let mut role_assignments = dsl::role_assignment .filter(dsl::identity_type.eq(identity_type.clone())) @@ -175,7 +174,7 @@ impl DataStore { Ok(role_assignments) }) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Fetches all of the externally-visible role assignments for the specified @@ -196,28 +195,19 @@ impl DataStore { opctx: &OpContext, authz_resource: &T, ) -> ListResultVec { - self.role_assignment_fetch_visible_conn( - opctx, - authz_resource, - self.pool_authorized(opctx).await?, - ) - .await + let conn = self.pool_connection_authorized(opctx).await?; + self.role_assignment_fetch_visible_conn(opctx, authz_resource, &conn) + .await } pub async fn role_assignment_fetch_visible_conn< T: authz::ApiResourceWithRoles + AuthorizedResource + Clone, - ConnErr, >( &self, opctx: &OpContext, authz_resource: &T, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), - ) -> ListResultVec - where - ConnErr: From + Send + 'static, - PoolError: From, - { + conn: &async_bb8_diesel::Connection, + ) -> ListResultVec { opctx.authorize(authz::Action::ReadPolicy, authz_resource).await?; let resource_type = authz_resource.resource_type(); let resource_id = authz_resource.resource_id(); @@ -231,9 +221,7 @@ impl DataStore { .select(RoleAssignment::as_select()) .load_async::(conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e.into(), ErrorHandler::Server) - }) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Removes all existing externally-visble role assignments on @@ -283,7 +271,7 @@ impl DataStore { // We might instead want to first-class the idea of Policies in the // database so that we can build up a whole new Policy in batches and // then flip the resource over to using it. - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { delete_old_query.execute_async(&conn).await?; @@ -292,8 +280,8 @@ impl DataStore { .await .map_err(|e| match e { TransactionError::CustomError(e) => e, - TransactionError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TransactionError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } }) } diff --git a/nexus/db-queries/src/db/datastore/saga.rs b/nexus/db-queries/src/db/datastore/saga.rs index 91e69e3fe5..2ec0c40799 100644 --- a/nexus/db-queries/src/db/datastore/saga.rs +++ b/nexus/db-queries/src/db/datastore/saga.rs @@ -6,7 +6,7 @@ use super::DataStore; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::Generation; use crate::db::pagination::paginated; @@ -30,11 +30,9 @@ impl DataStore { diesel::insert_into(dsl::saga) .values(saga.clone()) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_unauthorized().await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; Ok(()) } @@ -48,10 +46,10 @@ impl DataStore { // owning this saga. diesel::insert_into(dsl::saga_node_event) .values(event.clone()) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_unauthorized().await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict(ResourceType::SagaDbg, "Saga Event"), ) @@ -75,10 +73,10 @@ impl DataStore { .filter(dsl::adopt_generation.eq(current_adopt_generation)) .set(dsl::saga_state.eq(db::saga_types::SagaCachedState(new_state))) .check_if_exists::(saga_id) - .execute_and_check(self.pool()) + .execute_and_check(&*self.pool_connection_unauthorized().await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::SagaDbg, @@ -117,10 +115,10 @@ impl DataStore { steno::SagaCachedState::Done, ))) .filter(dsl::current_sec.eq(*sec_id)) - .load_async(self.pool()) + .load_async(&*self.pool_connection_unauthorized().await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::SagaDbg, @@ -138,10 +136,12 @@ impl DataStore { use db::schema::saga_node_event::dsl; paginated(dsl::saga_node_event, dsl::saga_id, &pagparams) .filter(dsl::saga_id.eq(id)) - .load_async::(self.pool()) + .load_async::( + &*self.pool_connection_unauthorized().await?, + ) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::SagaDbg, diff --git a/nexus/db-queries/src/db/datastore/service.rs b/nexus/db-queries/src/db/datastore/service.rs index b2c8505fea..40bf250abe 100644 --- a/nexus/db-queries/src/db/datastore/service.rs +++ b/nexus/db-queries/src/db/datastore/service.rs @@ -10,16 +10,14 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::identity::Asset; use crate::db::model::Service; use crate::db::model::Sled; use crate::db::pagination::paginated; use crate::db::pool::DbConnection; -use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::PoolError; use chrono::Utc; use diesel::prelude::*; use diesel::upsert::excluded; @@ -39,20 +37,16 @@ impl DataStore { opctx: &OpContext, service: Service, ) -> CreateResult { - let conn = self.pool_authorized(opctx).await?; - self.service_upsert_conn(conn, service).await + let conn = self.pool_connection_authorized(opctx).await?; + self.service_upsert_conn(&conn, service).await } /// Stores a new service in the database (using an existing db connection). - pub(crate) async fn service_upsert_conn( + pub(crate) async fn service_upsert_conn( &self, - conn: &(impl AsyncConnection + Sync), + conn: &async_bb8_diesel::Connection, service: Service, - ) -> CreateResult - where - ConnError: From + Send + 'static, - PoolError: From, - { + ) -> CreateResult { use db::schema::service::dsl; let service_id = service.id(); @@ -78,15 +72,13 @@ impl DataStore { type_name: ResourceType::Sled, lookup_type: LookupType::ById(sled_id), }, - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::Service, - &service_id.to_string(), - ), - ) - } + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Service, + &service_id.to_string(), + ), + ), }) } @@ -102,8 +94,8 @@ impl DataStore { paginated(dsl::service, dsl::id, pagparams) .filter(dsl::kind.eq(kind)) .select(Service::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs index ed2b97257e..5e909b84c4 100644 --- a/nexus/db-queries/src/db/datastore/silo.rs +++ b/nexus/db-queries/src/db/datastore/silo.rs @@ -10,8 +10,8 @@ use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::datastore::RunnableQuery; -use crate::db::error::diesel_pool_result_optional; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::diesel_result_optional; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::fixed_data::silo::{DEFAULT_SILO, INTERNAL_SILO}; @@ -24,7 +24,6 @@ use crate::db::pagination::paginated; use crate::db::pool::DbConnection; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::PoolError; use chrono::Utc; use diesel::prelude::*; use nexus_db_model::Certificate; @@ -66,11 +65,9 @@ impl DataStore { .values([&*DEFAULT_SILO, &*INTERNAL_SILO]) .on_conflict(dsl::id) .do_nothing() - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; info!(opctx.log, "created {} built-in silos", count); self.virtual_provisioning_collection_create( @@ -126,9 +123,9 @@ impl DataStore { new_silo_dns_names: &[String], dns_update: DnsVersionUpdateBuilder, ) -> CreateResult { - let conn = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; self.silo_create_conn( - conn, + &conn, opctx, nexus_opctx, new_silo_params, @@ -138,27 +135,15 @@ impl DataStore { .await } - pub async fn silo_create_conn( + pub async fn silo_create_conn( &self, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + conn: &async_bb8_diesel::Connection, opctx: &OpContext, nexus_opctx: &OpContext, new_silo_params: params::SiloCreate, new_silo_dns_names: &[String], dns_update: DnsVersionUpdateBuilder, - ) -> CreateResult - where - ConnErr: From + Send + 'static, - PoolError: From, - TransactionError: From, - - CalleeConnErr: From + Send + 'static, - PoolError: From, - TransactionError: From, - async_bb8_diesel::Connection: - AsyncConnection, - { + ) -> CreateResult { let silo_id = Uuid::new_v4(); let silo_group_id = Uuid::new_v4(); @@ -220,8 +205,8 @@ impl DataStore { .get_result_async(&conn) .await .map_err(|e| { - public_error_from_diesel_pool( - e.into(), + public_error_from_diesel( + e, ErrorHandler::Conflict( ResourceType::Silo, new_silo_params.identity.name.as_str(), @@ -276,8 +261,8 @@ impl DataStore { .await .map_err(|e| match e { TransactionError::CustomError(e) => e, - TransactionError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TransactionError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } }) } @@ -294,9 +279,9 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::discoverable.eq(true)) .select(Silo::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn silos_list( @@ -325,9 +310,9 @@ impl DataStore { query .select(Silo::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn silo_delete( @@ -348,19 +333,21 @@ impl DataStore { use db::schema::silo_user; use db::schema::silo_user_password_hash; + let conn = self.pool_connection_authorized(opctx).await?; + // Make sure there are no projects present within this silo. let id = authz_silo.id(); let rcgen = db_silo.rcgen; - let project_found = diesel_pool_result_optional( + let project_found = diesel_result_optional( project::dsl::project .filter(project::dsl::silo_id.eq(id)) .filter(project::dsl::time_deleted.is_null()) .select(project::dsl::id) .limit(1) - .first_async::(self.pool_authorized(opctx).await?) + .first_async::(&*conn) .await, ) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if project_found.is_some() { return Err(Error::InvalidRequest { @@ -371,48 +358,47 @@ impl DataStore { let now = Utc::now(); type TxnError = TransactionError; - self.pool_authorized(opctx) - .await? - .transaction_async(|conn| async move { - let updated_rows = diesel::update(silo::dsl::silo) - .filter(silo::dsl::time_deleted.is_null()) - .filter(silo::dsl::id.eq(id)) - .filter(silo::dsl::rcgen.eq(rcgen)) - .set(silo::dsl::time_deleted.eq(now)) - .execute_async(&conn) - .await - .map_err(|e| { - public_error_from_diesel_pool( - PoolError::from(e), - ErrorHandler::NotFoundByResource(authz_silo), - ) - })?; - - if updated_rows == 0 { - return Err(TxnError::CustomError(Error::InvalidRequest { - message: "silo deletion failed due to concurrent modification" + conn.transaction_async(|conn| async move { + let updated_rows = diesel::update(silo::dsl::silo) + .filter(silo::dsl::time_deleted.is_null()) + .filter(silo::dsl::id.eq(id)) + .filter(silo::dsl::rcgen.eq(rcgen)) + .set(silo::dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_silo), + ) + })?; + + if updated_rows == 0 { + return Err(TxnError::CustomError(Error::InvalidRequest { + message: + "silo deletion failed due to concurrent modification" .to_string(), - })); - } + })); + } - self.virtual_provisioning_collection_delete_on_connection( - &conn, - id, - ).await?; + self.virtual_provisioning_collection_delete_on_connection( + &conn, id, + ) + .await?; - self.dns_update(dns_opctx, &conn, dns_update).await?; + self.dns_update(dns_opctx, &conn, dns_update).await?; - info!(opctx.log, "deleted silo {}", id); + info!(opctx.log, "deleted silo {}", id); - Ok(()) - }) - .await - .map_err(|e| match e { - TxnError::CustomError(e) => e, - TxnError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) - } - })?; + Ok(()) + }) + .await + .map_err(|e| match e { + TxnError::CustomError(e) => e, + TxnError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + })?; // TODO-correctness This needs to happen in a saga or some other // mechanism that ensures it happens even if we crash at this point. @@ -429,9 +415,9 @@ impl DataStore { .select(silo_user::dsl::id), ), ) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; debug!( opctx.log, @@ -442,11 +428,9 @@ impl DataStore { .filter(silo_user::dsl::silo_id.eq(id)) .filter(silo_user::dsl::time_deleted.is_null()) .set(silo_user::dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; debug!( opctx.log, @@ -464,10 +448,10 @@ impl DataStore { .select(silo_group::dsl::id), ), ) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) })?; debug!( @@ -480,11 +464,9 @@ impl DataStore { .filter(silo_group::dsl::silo_id.eq(id)) .filter(silo_group::dsl::time_deleted.is_null()) .set(silo_group::dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; debug!( opctx.log, @@ -498,11 +480,9 @@ impl DataStore { .filter(idp_dsl::silo_id.eq(id)) .filter(idp_dsl::time_deleted.is_null()) .set(idp_dsl::time_deleted.eq(Utc::now())) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; debug!(opctx.log, "deleted {} silo IdPs for silo {}", updated_rows, id); @@ -512,11 +492,9 @@ impl DataStore { .filter(saml_idp_dsl::silo_id.eq(id)) .filter(saml_idp_dsl::time_deleted.is_null()) .set(saml_idp_dsl::time_deleted.eq(Utc::now())) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; debug!( opctx.log, @@ -530,11 +508,9 @@ impl DataStore { .filter(cert_dsl::silo_id.eq(id)) .filter(cert_dsl::time_deleted.is_null()) .set(cert_dsl::time_deleted.eq(Utc::now())) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; debug!(opctx.log, "deleted {} silo IdPs for silo {}", updated_rows, id); diff --git a/nexus/db-queries/src/db/datastore/silo_group.rs b/nexus/db-queries/src/db/datastore/silo_group.rs index 0261dc5542..d13986bb2d 100644 --- a/nexus/db-queries/src/db/datastore/silo_group.rs +++ b/nexus/db-queries/src/db/datastore/silo_group.rs @@ -9,7 +9,7 @@ use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::datastore::RunnableQuery; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::model::SiloGroup; @@ -56,11 +56,9 @@ impl DataStore { DataStore::silo_group_ensure_query(opctx, authz_silo, silo_group) .await? - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; Ok(self .silo_group_optional_lookup(opctx, authz_silo, external_id) @@ -83,10 +81,10 @@ impl DataStore { .filter(dsl::external_id.eq(external_id)) .filter(dsl::time_deleted.is_null()) .select(db::model::SiloGroup::as_select()) - .first_async(self.pool_authorized(opctx).await?) + .first_async(&*self.pool_connection_authorized(opctx).await?) .await .optional() - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn silo_group_membership_for_user( @@ -101,9 +99,9 @@ impl DataStore { dsl::silo_group_membership .filter(dsl::silo_user_id.eq(silo_user_id)) .select(SiloGroupMembership::as_returning()) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn silo_groups_for_self( @@ -125,9 +123,9 @@ impl DataStore { .filter(sgm::silo_user_id.eq(actor.actor_id())) .filter(sg::time_deleted.is_null()) .select(SiloGroup::as_returning()) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Update a silo user's group membership: @@ -147,7 +145,7 @@ impl DataStore { ) -> UpdateResult<()> { opctx.authorize(authz::Action::Modify, authz_silo_user).await?; - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { use db::schema::silo_group_membership::dsl; @@ -166,7 +164,7 @@ impl DataStore { .iter() .map(|group_id| db::model::SiloGroupMembership { silo_group_id: *group_id, - silo_user_id: silo_user_id, + silo_user_id, }) .collect(); @@ -178,7 +176,7 @@ impl DataStore { Ok(()) }) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn silo_group_delete( @@ -197,7 +195,7 @@ impl DataStore { let group_id = authz_silo_group.id(); - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { use db::schema::silo_group_membership; @@ -240,10 +238,9 @@ impl DataStore { id )), - TxnError::Pool(pool_error) => public_error_from_diesel_pool( - pool_error, - ErrorHandler::Server, - ), + TxnError::Connection(error) => { + public_error_from_diesel(error, ErrorHandler::Server) + } }) } @@ -260,8 +257,10 @@ impl DataStore { .filter(dsl::silo_id.eq(authz_silo.id())) .filter(dsl::time_deleted.is_null()) .select(SiloGroup::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/silo_user.rs b/nexus/db-queries/src/db/datastore/silo_user.rs index e0fcf6c469..6084f8c2ab 100644 --- a/nexus/db-queries/src/db/datastore/silo_user.rs +++ b/nexus/db-queries/src/db/datastore/silo_user.rs @@ -10,7 +10,7 @@ use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::datastore::IdentityMetadataCreateParams; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::Name; use crate::db::model::Silo; @@ -53,13 +53,14 @@ impl DataStore { use db::schema::silo_user::dsl; let silo_user_external_id = silo_user.external_id.clone(); + let conn = self.pool_connection_unauthorized().await?; diesel::insert_into(dsl::silo_user) .values(silo_user) .returning(SiloUser::as_returning()) - .get_result_async(self.pool()) + .get_result_async(&*conn) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SiloUser, @@ -91,7 +92,7 @@ impl DataStore { // TODO-robustness We might consider the RFD 192 "rcgen" pattern as well // so that people can't, say, login while we do this. let authz_silo_user_id = authz_silo_user.id(); - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|mut conn| async move { // Delete the user record. @@ -148,7 +149,7 @@ impl DataStore { }) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_silo_user), ) @@ -176,11 +177,11 @@ impl DataStore { .filter(dsl::external_id.eq(external_id.to_string())) .filter(dsl::time_deleted.is_null()) .select(SiloUser::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })? + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .pop() .map(|db_silo_user| { let authz_silo_user = authz::SiloUser::new( @@ -208,9 +209,11 @@ impl DataStore { .filter(silo_id.eq(authz_silo_user_list.silo().id())) .filter(time_deleted.is_null()) .select(SiloUser::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn silo_group_users_list( @@ -237,9 +240,11 @@ impl DataStore { ), )) .select(SiloUser::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Updates or deletes the password hash for a given Silo user @@ -280,18 +285,18 @@ impl DataStore { .on_conflict(dsl::silo_user_id) .do_update() .set(SiloUserPasswordUpdate::new(hash_for_update)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) })?; } else { diesel::delete(dsl::silo_user_password_hash) .filter(dsl::silo_user_id.eq(authz_silo_user.id())) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) })?; } @@ -323,12 +328,10 @@ impl DataStore { .filter(dsl::silo_user_id.eq(authz_silo_user.id())) .select(SiloUserPasswordHash::as_select()) .load_async::( - self.pool_authorized(opctx).await?, + &*self.pool_connection_authorized(opctx).await?, ) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })? + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .pop()) } @@ -341,9 +344,11 @@ impl DataStore { opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; paginated(dsl::user_builtin, dsl::name, pagparams) .select(UserBuiltin::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Load built-in users into the database @@ -383,11 +388,9 @@ impl DataStore { .values(builtin_users) .on_conflict(dsl::id) .do_nothing() - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; info!(opctx.log, "created {} built-in users", count); Ok(()) @@ -410,11 +413,9 @@ impl DataStore { .values(users) .on_conflict(dsl::id) .do_nothing() - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; info!(opctx.log, "created {} silo users", count); Ok(()) @@ -437,11 +438,9 @@ impl DataStore { dsl::role_name, )) .do_nothing() - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; info!(opctx.log, "created {} silo user role assignments", count); Ok(()) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index a70ec26d8c..ec6cca0071 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -8,7 +8,7 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::identity::Asset; @@ -46,10 +46,10 @@ impl DataStore { dsl::reservoir_size.eq(sled.reservoir_size), )) .returning(Sled::as_returning()) - .get_result_async(self.pool()) + .get_result_async(&*self.pool_connection_unauthorized().await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::Sled, @@ -68,9 +68,9 @@ impl DataStore { use db::schema::sled::dsl; paginated(dsl::sled, dsl::id, pagparams) .select(Sled::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn sled_reservation_create( @@ -87,7 +87,7 @@ impl DataStore { } type TxnError = TransactionError; - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { use db::schema::sled_resource::dsl as resource_dsl; @@ -183,8 +183,8 @@ impl DataStore { "No sleds can fit the requested instance", ) } - TxnError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TxnError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } }) } @@ -197,11 +197,9 @@ impl DataStore { use db::schema::sled_resource::dsl as resource_dsl; diesel::delete(resource_dsl::sled_resource) .filter(resource_dsl::id.eq(resource_id)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; Ok(()) } } diff --git a/nexus/db-queries/src/db/datastore/sled_instance.rs b/nexus/db-queries/src/db/datastore/sled_instance.rs index 9ba6861cec..dbdd696d70 100644 --- a/nexus/db-queries/src/db/datastore/sled_instance.rs +++ b/nexus/db-queries/src/db/datastore/sled_instance.rs @@ -3,7 +3,7 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::pagination::paginated; use async_bb8_diesel::AsyncRunQueryDsl; @@ -25,8 +25,10 @@ impl DataStore { paginated(dsl::sled_instance, dsl::id, &pagparams) .filter(dsl::active_sled_id.eq(authz_sled.id())) .select(SledInstance::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/snapshot.rs b/nexus/db-queries/src/db/datastore/snapshot.rs index d8db6d72a4..29fbb38e88 100644 --- a/nexus/db-queries/src/db/datastore/snapshot.rs +++ b/nexus/db-queries/src/db/datastore/snapshot.rs @@ -10,7 +10,7 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::Generation; use crate::db::model::Name; @@ -63,7 +63,7 @@ impl DataStore { let project_id = snapshot.project_id; let snapshot: Snapshot = self - .pool_authorized(opctx) + .pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { use db::schema::snapshot::dsl; @@ -157,16 +157,13 @@ impl DataStore { } } AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Server, - ) + public_error_from_diesel(e, ErrorHandler::Server) } }, }, - TxnError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TxnError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } })?; @@ -203,10 +200,10 @@ impl DataStore { .filter(dsl::gen.eq(old_gen)) .set((dsl::state.eq(new_state), dsl::gen.eq(next_gen))) .returning(Snapshot::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_snapshot), ) @@ -235,9 +232,9 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::project_id.eq(authz_project.id())) .select(Snapshot::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn project_delete_snapshot( @@ -273,11 +270,9 @@ impl DataStore { dsl::state.eq(SnapshotState::Destroyed), )) .check_if_exists::(snapshot_id) - .execute_async(self.pool_authorized(&opctx).await?) + .execute_async(&*self.pool_connection_authorized(&opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if updated_rows == 0 { // Either: diff --git a/nexus/db-queries/src/db/datastore/ssh_key.rs b/nexus/db-queries/src/db/datastore/ssh_key.rs index 622a54d740..c925903e12 100644 --- a/nexus/db-queries/src/db/datastore/ssh_key.rs +++ b/nexus/db-queries/src/db/datastore/ssh_key.rs @@ -8,7 +8,7 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::identity::Resource; use crate::db::model::Name; @@ -48,9 +48,9 @@ impl DataStore { .filter(dsl::silo_user_id.eq(authz_user.id())) .filter(dsl::time_deleted.is_null()) .select(SshKey::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Create a new SSH public key for a user. @@ -68,10 +68,10 @@ impl DataStore { diesel::insert_into(dsl::ssh_key) .values(ssh_key) .returning(SshKey::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict(ResourceType::SshKey, &name), ) @@ -92,10 +92,10 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .set(dsl::time_deleted.eq(Utc::now())) .check_if_exists::(authz_ssh_key.id()) - .execute_and_check(self.pool_authorized(opctx).await?) + .execute_and_check(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_ssh_key), ) diff --git a/nexus/db-queries/src/db/datastore/switch.rs b/nexus/db-queries/src/db/datastore/switch.rs index 56cfb9a96a..148f4577de 100644 --- a/nexus/db-queries/src/db/datastore/switch.rs +++ b/nexus/db-queries/src/db/datastore/switch.rs @@ -2,7 +2,7 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::identity::Asset; use crate::db::model::Switch; @@ -20,6 +20,8 @@ impl DataStore { /// Stores a new switch in the database. pub async fn switch_upsert(&self, switch: Switch) -> CreateResult { use db::schema::switch::dsl; + + let conn = self.pool_connection_unauthorized().await?; diesel::insert_into(dsl::switch) .values(switch.clone()) .on_conflict(dsl::id) @@ -29,10 +31,10 @@ impl DataStore { dsl::rack_id.eq(switch.rack_id), )) .returning(Switch::as_returning()) - .get_result_async(self.pool()) + .get_result_async(&*conn) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::Switch, @@ -51,8 +53,8 @@ impl DataStore { use db::schema::switch::dsl; paginated(dsl::switch, dsl::id, pagparams) .select(Switch::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/switch_interface.rs b/nexus/db-queries/src/db/datastore/switch_interface.rs index 5c26dc5431..498064ce37 100644 --- a/nexus/db-queries/src/db/datastore/switch_interface.rs +++ b/nexus/db-queries/src/db/datastore/switch_interface.rs @@ -9,14 +9,12 @@ use crate::db; use crate::db::datastore::address_lot::{ ReserveBlockError, ReserveBlockTxnError, }; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::model::LoopbackAddress; use crate::db::pagination::paginated; -use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, ConnectionError, PoolError, -}; +use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, ConnectionError}; use diesel::result::Error as DieselError; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use ipnetwork::IpNetwork; @@ -44,14 +42,14 @@ impl DataStore { type TxnError = TransactionError; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let inet = IpNetwork::new(params.address, params.mask) .map_err(|_| Error::invalid_request("invalid address"))?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { let lot_id = authz_address_lot.id(); let (block, rsvd_block) = crate::db::datastore::address_lot::try_reserve_block( @@ -67,7 +65,9 @@ impl DataStore { LoopbackAddressCreateError::ReserveBlock(err), ) } - ReserveBlockTxnError::Pool(err) => TxnError::Pool(err), + ReserveBlockTxnError::Connection(err) => { + TxnError::Connection(err) + } })?; // Address block reserved, now create the loopback address. @@ -103,17 +103,17 @@ impl DataStore { ReserveBlockError::AddressNotInLot, ), ) => Error::invalid_request("address not in lot"), - TxnError::Pool(e) => match e { - PoolError::Connection(ConnectionError::Query( - DieselError::DatabaseError(_, _), - )) => public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::LoopbackAddress, - &format!("lo {}", inet), - ), - ), - _ => public_error_from_diesel_pool(e, ErrorHandler::Server), + TxnError::Connection(e) => match e { + ConnectionError::Query(DieselError::DatabaseError(_, _)) => { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::LoopbackAddress, + &format!("lo {}", inet), + ), + ) + } + _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) } @@ -128,11 +128,11 @@ impl DataStore { let id = authz_loopback_address.id(); - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { let la = diesel::delete(dsl::loopback_address) .filter(dsl::id.eq(id)) .returning(LoopbackAddress::as_returning()) @@ -147,7 +147,7 @@ impl DataStore { Ok(()) }) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn loopback_address_get( @@ -160,15 +160,15 @@ impl DataStore { let id = authz_loopback_address.id(); - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; loopback_dsl::loopback_address .filter(loopback_address::id.eq(id)) .select(LoopbackAddress::as_select()) .limit(1) - .first_async::(pool) + .first_async::(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn loopback_address_list( @@ -180,8 +180,8 @@ impl DataStore { paginated(dsl::loopback_address, dsl::id, &pagparams) .select(LoopbackAddress::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index 33dfd56359..940fedb473 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -9,7 +9,7 @@ use crate::db::datastore::address_lot::{ ReserveBlockError, ReserveBlockTxnError, }; use crate::db::datastore::UpdatePrecondition; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::model::{ @@ -20,9 +20,7 @@ use crate::db::model::{ SwitchVlanInterfaceConfig, }; use crate::db::pagination::paginated; -use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, ConnectionError, PoolError, -}; +use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, ConnectionError}; use diesel::result::Error as DieselError; use diesel::{ ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl, @@ -128,11 +126,11 @@ impl DataStore { } type TxnError = TransactionError; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { // create the top level port settings object let port_settings = SwitchPortSettings::new(¶ms.identity); @@ -371,7 +369,7 @@ impl DataStore { SwitchPortSettingsCreateError::ReserveBlock(err) ) } - ReserveBlockTxnError::Pool(err) => TxnError::Pool(err), + ReserveBlockTxnError::Connection(err) => TxnError::Connection(err), })?; address_config.push(SwitchPortAddressConfig::new( @@ -418,17 +416,17 @@ impl DataStore { ReserveBlockError::AddressNotInLot ) ) => Error::invalid_request("address not in lot"), - TxnError::Pool(e) => match e { - PoolError::Connection(ConnectionError::Query( + TxnError::Connection(e) => match e { + ConnectionError::Query( DieselError::DatabaseError(_, _), - )) => public_error_from_diesel_pool( + ) => public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SwitchPortSettings, params.identity.name.as_str(), ), ), - _ => public_error_from_diesel_pool(e, ErrorHandler::Server), + _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) } @@ -446,7 +444,7 @@ impl DataStore { } type TxnError = TransactionError; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let selector = match ¶ms.port_settings { None => return Err(Error::invalid_request("name or id required")), @@ -455,7 +453,7 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { use db::schema::switch_port_settings; let id = match selector { @@ -601,15 +599,15 @@ impl DataStore { SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound) => { Error::invalid_request("port settings not found") } - TxnError::Pool(e) => match e { - PoolError::Connection(ConnectionError::Query( + TxnError::Connection(e) => match e { + ConnectionError::Query( DieselError::DatabaseError(_, _), - )) => { + ) => { let name = match ¶ms.port_settings { Some(name_or_id) => name_or_id.to_string(), None => String::new(), }; - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SwitchPortSettings, @@ -617,7 +615,7 @@ impl DataStore { ), ) }, - _ => public_error_from_diesel_pool(e, ErrorHandler::Server), + _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) } @@ -641,9 +639,9 @@ impl DataStore { } .filter(dsl::time_deleted.is_null()) .select(SwitchPortSettings::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn switch_port_settings_get( @@ -657,11 +655,11 @@ impl DataStore { } type TxnError = TransactionError; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { // get the top level port settings object use db::schema::switch_port_settings::dsl as port_settings_dsl; @@ -806,12 +804,12 @@ impl DataStore { SwitchPortSettingsGetError::NotFound(name)) => { Error::not_found_by_name(ResourceType::SwitchPortSettings, &name) } - TxnError::Pool(e) => match e { - PoolError::Connection(ConnectionError::Query( + TxnError::Connection(e) => match e { + ConnectionError::Query( DieselError::DatabaseError(_, _), - )) => { + ) => { let name = name_or_id.to_string(); - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SwitchPortSettings, @@ -819,7 +817,7 @@ impl DataStore { ), ) }, - _ => public_error_from_diesel_pool(e, ErrorHandler::Server), + _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) } @@ -839,7 +837,7 @@ impl DataStore { } type TxnError = TransactionError; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let switch_port = SwitchPort::new( rack_id, switch_location.to_string(), @@ -848,7 +846,7 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { use db::schema::rack; use db::schema::rack::dsl as rack_dsl; rack_dsl::rack @@ -880,17 +878,20 @@ impl DataStore { TxnError::CustomError(SwitchPortCreateError::RackNotFound) => { Error::invalid_request("rack not found") } - TxnError::Pool(e) => match e { - PoolError::Connection(ConnectionError::Query( - DieselError::DatabaseError(_, _), - )) => public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::SwitchPort, - &format!("{}/{}/{}", rack_id, &switch_location, &port,), - ), - ), - _ => public_error_from_diesel_pool(e, ErrorHandler::Server), + TxnError::Connection(e) => match e { + ConnectionError::Query(DieselError::DatabaseError(_, _)) => { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::SwitchPort, + &format!( + "{}/{}/{}", + rack_id, &switch_location, &port, + ), + ), + ) + } + _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) } @@ -908,11 +909,11 @@ impl DataStore { } type TxnError = TransactionError; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - pool.transaction_async(|conn| async move { + conn.transaction_async(|conn| async move { use db::schema::switch_port; use db::schema::switch_port::dsl as switch_port_dsl; @@ -957,8 +958,8 @@ impl DataStore { TxnError::CustomError(SwitchPortDeleteError::ActiveSettings) => { Error::invalid_request("must clear port settings first") } - TxnError::Pool(e) => { - public_error_from_diesel_pool(e, ErrorHandler::Server) + TxnError::Connection(e) => { + public_error_from_diesel(e, ErrorHandler::Server) } }) } @@ -972,9 +973,9 @@ impl DataStore { paginated(dsl::switch_port, dsl::id, pagparams) .select(SwitchPort::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn switch_port_get( @@ -985,15 +986,15 @@ impl DataStore { use db::schema::switch_port; use db::schema::switch_port::dsl as switch_port_dsl; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; switch_port_dsl::switch_port .filter(switch_port::id.eq(id)) .select(SwitchPort::as_select()) .limit(1) - .first_async::(pool) + .first_async::(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn switch_port_set_settings_id( @@ -1006,17 +1007,17 @@ impl DataStore { use db::schema::switch_port; use db::schema::switch_port::dsl as switch_port_dsl; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; match current { UpdatePrecondition::DontCare => { diesel::update(switch_port_dsl::switch_port) .filter(switch_port::id.eq(switch_port_id)) .set(switch_port::port_settings_id.eq(port_settings_id)) - .execute_async(pool) + .execute_async(&*conn) .await .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) })?; } UpdatePrecondition::Null => { @@ -1024,10 +1025,10 @@ impl DataStore { .filter(switch_port::id.eq(switch_port_id)) .filter(switch_port::port_settings_id.is_null()) .set(switch_port::port_settings_id.eq(port_settings_id)) - .execute_async(pool) + .execute_async(&*conn) .await .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) })?; } UpdatePrecondition::Value(current_id) => { @@ -1035,10 +1036,10 @@ impl DataStore { .filter(switch_port::id.eq(switch_port_id)) .filter(switch_port::port_settings_id.eq(current_id)) .set(switch_port::port_settings_id.eq(port_settings_id)) - .execute_async(pool) + .execute_async(&*conn) .await .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) })?; } } @@ -1056,7 +1057,7 @@ impl DataStore { use db::schema::switch_port; use db::schema::switch_port::dsl as switch_port_dsl; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let id: Uuid = switch_port_dsl::switch_port .filter(switch_port::rack_id.eq(rack_id)) .filter( @@ -1065,7 +1066,7 @@ impl DataStore { .filter(switch_port::port_name.eq(port_name.to_string())) .select(switch_port::id) .limit(1) - .first_async::(pool) + .first_async::(&*conn) .await .map_err(|_| { Error::not_found_by_name(ResourceType::SwitchPort, &port_name) @@ -1082,7 +1083,7 @@ impl DataStore { use db::schema::switch_port_settings; use db::schema::switch_port_settings::dsl as port_settings_dsl; - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let db_name = name.to_string(); let id = port_settings_dsl::switch_port_settings @@ -1090,7 +1091,7 @@ impl DataStore { .filter(switch_port_settings::name.eq(db_name)) .select(switch_port_settings::id) .limit(1) - .first_async::(pool) + .first_async::(&*conn) .await .map_err(|_| { Error::not_found_by_name( @@ -1122,8 +1123,10 @@ impl DataStore { // pagination in the future, or maybe a way to constrain the query to // a rack? .limit(64) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/update.rs b/nexus/db-queries/src/db/datastore/update.rs index 851ee66bd9..5a3e3b27e4 100644 --- a/nexus/db-queries/src/db/datastore/update.rs +++ b/nexus/db-queries/src/db/datastore/update.rs @@ -9,7 +9,7 @@ use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::error::{ - public_error_from_diesel_pool, ErrorHandler, TransactionError, + public_error_from_diesel, ErrorHandler, TransactionError, }; use crate::db::model::{ ComponentUpdate, SemverVersion, SystemUpdate, UpdateArtifact, @@ -42,9 +42,9 @@ impl DataStore { .do_update() .set(artifact.clone()) .returning(UpdateArtifact::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn update_artifact_hard_delete_outdated( @@ -60,10 +60,10 @@ impl DataStore { use db::schema::update_artifact::dsl; diesel::delete(dsl::update_artifact) .filter(dsl::targets_role_version.lt(current_targets_role_version)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map(|_rows_deleted| ()) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) .internal_context("deleting outdated available artifacts") } @@ -84,10 +84,10 @@ impl DataStore { // to add more metadata to this model .set(time_modified.eq(Utc::now())) .returning(SystemUpdate::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SystemUpdate, @@ -112,10 +112,10 @@ impl DataStore { system_update .filter(version.eq(target)) .select(SystemUpdate::as_select()) - .first_async(self.pool_authorized(opctx).await?) + .first_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::SystemUpdate, @@ -141,7 +141,7 @@ impl DataStore { let version_string = update.version.to_string(); - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { let db_update = diesel::insert_into(component_update::table) @@ -164,7 +164,7 @@ impl DataStore { .await .map_err(|e| match e { TransactionError::CustomError(e) => e, - TransactionError::Pool(e) => public_error_from_diesel_pool( + TransactionError::Connection(e) => public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::ComponentUpdate, @@ -186,9 +186,9 @@ impl DataStore { paginated(system_update, id, pagparams) .select(SystemUpdate::as_select()) .order(version.desc()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn system_update_components_list( @@ -205,9 +205,9 @@ impl DataStore { .inner_join(join_table::table) .filter(join_table::columns::system_update_id.eq(system_update_id)) .select(ComponentUpdate::as_select()) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn create_updateable_component( @@ -226,10 +226,10 @@ impl DataStore { diesel::insert_into(updateable_component) .values(component.clone()) .returning(UpdateableComponent::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::UpdateableComponent, @@ -250,9 +250,9 @@ impl DataStore { paginated(updateable_component, id, pagparams) .select(UpdateableComponent::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn lowest_component_system_version( @@ -266,9 +266,9 @@ impl DataStore { updateable_component .select(system_version) .order(system_version.asc()) - .first_async(self.pool_authorized(opctx).await?) + .first_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn highest_component_system_version( @@ -282,9 +282,9 @@ impl DataStore { updateable_component .select(system_version) .order(system_version.desc()) - .first_async(self.pool_authorized(opctx).await?) + .first_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn create_update_deployment( @@ -299,10 +299,10 @@ impl DataStore { diesel::insert_into(update_deployment) .values(deployment.clone()) .returning(UpdateDeployment::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::UpdateDeployment, @@ -330,10 +330,10 @@ impl DataStore { time_modified.eq(diesel::dsl::now), )) .returning(UpdateDeployment::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::UpdateDeployment, @@ -354,9 +354,9 @@ impl DataStore { paginated(update_deployment, id, pagparams) .select(UpdateDeployment::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn latest_update_deployment( @@ -370,8 +370,8 @@ impl DataStore { update_deployment .select(UpdateDeployment::as_returning()) .order(time_created.desc()) - .first_async(self.pool_authorized(opctx).await?) + .first_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs index 404b071ad9..18ff58735e 100644 --- a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs +++ b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs @@ -7,13 +7,13 @@ use super::DataStore; use crate::context::OpContext; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::ByteCount; use crate::db::model::VirtualProvisioningCollection; use crate::db::pool::DbConnection; use crate::db::queries::virtual_provisioning_collection_update::VirtualProvisioningCollectionUpdate; -use async_bb8_diesel::{AsyncRunQueryDsl, PoolError}; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use omicron_common::api::external::{DeleteResult, Error}; use uuid::Uuid; @@ -46,26 +46,19 @@ impl DataStore { opctx: &OpContext, virtual_provisioning_collection: VirtualProvisioningCollection, ) -> Result, Error> { - let pool = self.pool_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; self.virtual_provisioning_collection_create_on_connection( - pool, + &conn, virtual_provisioning_collection, ) .await } - pub(crate) async fn virtual_provisioning_collection_create_on_connection< - ConnErr, - >( + pub(crate) async fn virtual_provisioning_collection_create_on_connection( &self, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + conn: &async_bb8_diesel::Connection, virtual_provisioning_collection: VirtualProvisioningCollection, - ) -> Result, Error> - where - ConnErr: From + Send + 'static, - PoolError: From, - { + ) -> Result, Error> { use db::schema::virtual_provisioning_collection::dsl; let provisions: Vec = @@ -75,10 +68,7 @@ impl DataStore { .get_results_async(conn) .await .map_err(|e| { - public_error_from_diesel_pool( - PoolError::from(e), - ErrorHandler::Server, - ) + public_error_from_diesel(e, ErrorHandler::Server) })?; self.virtual_provisioning_collection_producer .append_all_metrics(&provisions)?; @@ -96,10 +86,12 @@ impl DataStore { dsl::virtual_provisioning_collection .find(id) .select(VirtualProvisioningCollection::as_select()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) .await .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) })?; Ok(virtual_provisioning_collection) } @@ -110,24 +102,17 @@ impl DataStore { opctx: &OpContext, id: Uuid, ) -> DeleteResult { - let pool = self.pool_authorized(opctx).await?; - self.virtual_provisioning_collection_delete_on_connection(pool, id) + let conn = self.pool_connection_authorized(opctx).await?; + self.virtual_provisioning_collection_delete_on_connection(&conn, id) .await } /// Delete a [`VirtualProvisioningCollection`] object. - pub(crate) async fn virtual_provisioning_collection_delete_on_connection< - ConnErr, - >( + pub(crate) async fn virtual_provisioning_collection_delete_on_connection( &self, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), + conn: &async_bb8_diesel::Connection, id: Uuid, - ) -> DeleteResult - where - ConnErr: From + Send + 'static, - PoolError: From, - { + ) -> DeleteResult { use db::schema::virtual_provisioning_collection::dsl; // NOTE: We don't really need to extract the value we're deleting from @@ -138,12 +123,7 @@ impl DataStore { .returning(VirtualProvisioningCollection::as_select()) .get_result_async(conn) .await - .map_err(|e| { - public_error_from_diesel_pool( - PoolError::from(e), - ErrorHandler::Server, - ) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; assert!( collection.is_empty(), "Collection deleted while non-empty: {collection:?}" @@ -209,11 +189,9 @@ impl DataStore { project_id, storage_type, ) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; self.virtual_provisioning_collection_producer .append_disk_metrics(&provisions)?; Ok(provisions) @@ -265,11 +243,9 @@ impl DataStore { disk_byte_diff, project_id, ) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; self.virtual_provisioning_collection_producer .append_disk_metrics(&provisions)?; Ok(provisions) @@ -288,11 +264,9 @@ impl DataStore { VirtualProvisioningCollectionUpdate::new_insert_instance( id, cpus_diff, ram_diff, project_id, ) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; self.virtual_provisioning_collection_producer .append_cpu_metrics(&provisions)?; Ok(provisions) @@ -311,11 +285,9 @@ impl DataStore { VirtualProvisioningCollectionUpdate::new_delete_instance( id, cpus_diff, ram_diff, project_id, ) - .get_results_async(self.pool_authorized(opctx).await?) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; self.virtual_provisioning_collection_producer .append_cpu_metrics(&provisions)?; Ok(provisions) diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index 901cf16f63..b3e82886de 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -6,7 +6,7 @@ use super::DataStore; use crate::db; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::identity::Asset; @@ -19,7 +19,6 @@ use async_bb8_diesel::AsyncRunQueryDsl; use async_bb8_diesel::OptionalExtension; use chrono::Utc; use diesel::prelude::*; -use diesel::OptionalExtension as DieselOptionalExtension; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; @@ -65,19 +64,18 @@ impl DataStore { crucible_targets }; - self.pool() - .transaction(move |conn| { + self.pool_connection_unauthorized() + .await? + .transaction_async(|conn| async move { let maybe_volume: Option = dsl::volume .filter(dsl::id.eq(volume.id())) .select(Volume::as_select()) - .first(conn) + .first_async(&conn) + .await .optional() .map_err(|e| { TxnError::CustomError(VolumeCreationError::Public( - public_error_from_diesel_pool( - e.into(), - ErrorHandler::Server, - ), + public_error_from_diesel(e, ErrorHandler::Server), )) })?; @@ -97,11 +95,12 @@ impl DataStore { .on_conflict(dsl::id) .do_nothing() .returning(Volume::as_returning()) - .get_result(conn) + .get_result_async(&conn) + .await .map_err(|e| { TxnError::CustomError(VolumeCreationError::Public( - public_error_from_diesel_pool( - e.into(), + public_error_from_diesel( + e, ErrorHandler::Conflict( ResourceType::Volume, volume.id().to_string().as_str(), @@ -124,11 +123,12 @@ impl DataStore { rs_dsl::volume_references .eq(rs_dsl::volume_references + 1), ) - .execute(conn) + .execute_async(&conn) + .await .map_err(|e| { TxnError::CustomError(VolumeCreationError::Public( - public_error_from_diesel_pool( - e.into(), + public_error_from_diesel( + e, ErrorHandler::Server, ), )) @@ -156,10 +156,10 @@ impl DataStore { dsl::volume .filter(dsl::id.eq(volume_id)) .select(Volume::as_select()) - .first_async::(self.pool()) + .first_async::(&*self.pool_connection_unauthorized().await?) .await .optional() - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Delete the volume if it exists. If it was already deleted, this is a @@ -169,10 +169,10 @@ impl DataStore { diesel::delete(dsl::volume) .filter(dsl::id.eq(volume_id)) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_unauthorized().await?) .await .map(|_| ()) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Checkout a copy of the Volume from the database. @@ -206,13 +206,15 @@ impl DataStore { // types that require it). The generation number (along with the // rest of the volume data) that was in the database is what is // returned to the caller. - self.pool() - .transaction(move |conn| { + self.pool_connection_unauthorized() + .await? + .transaction_async(|conn| async move { // Grab the volume in question. let volume = dsl::volume .filter(dsl::id.eq(volume_id)) .select(Volume::as_select()) - .get_result(conn)?; + .get_result_async(&conn) + .await?; // Turn the volume.data into the VolumeConstructionRequest let vcr: VolumeConstructionRequest = @@ -289,7 +291,8 @@ impl DataStore { diesel::update(volume_dsl::volume) .filter(volume_dsl::id.eq(volume_id)) .set(volume_dsl::data.eq(new_volume_data)) - .execute(conn)?; + .execute_async(&conn) + .await?; // This should update just one row. If it does // not, then something is terribly wrong in the @@ -332,10 +335,7 @@ impl DataStore { .await .map_err(|e| match e { TxnError::CustomError(VolumeGetError::DieselError(e)) => { - public_error_from_diesel_pool( - e.into(), - ErrorHandler::Server, - ) + public_error_from_diesel(e.into(), ErrorHandler::Server) } _ => { @@ -478,9 +478,9 @@ impl DataStore { Region::as_select(), Volume::as_select(), )) - .load_async(self.pool()) + .load_async(&*self.pool_connection_unauthorized().await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn read_only_resources_associated_with_volume( @@ -576,8 +576,9 @@ impl DataStore { // // TODO it would be nice to make this transaction_async, but I couldn't // get the async optional extension to work. - self.pool() - .transaction(move |conn| { + self.pool_connection_unauthorized() + .await? + .transaction_async(|conn| async move { // Grab the volume in question. If the volume record was already // hard-deleted, assume clean-up has occurred and return an empty // CrucibleResources. If the volume record was soft-deleted, then @@ -588,7 +589,8 @@ impl DataStore { let volume = volume_dsl::volume .filter(volume_dsl::id.eq(volume_id)) .select(Volume::as_select()) - .get_result(conn) + .get_result_async(&conn) + .await .optional()?; let volume = if let Some(v) = volume { @@ -643,10 +645,11 @@ impl DataStore { diesel::update(dsl::region_snapshot) .filter( dsl::snapshot_addr - .eq_any(&crucible_targets.read_only_targets), + .eq_any(crucible_targets.read_only_targets.clone()), ) .set(dsl::volume_references.eq(dsl::volume_references - 1)) - .execute(conn)?; + .execute_async(&conn) + .await?; // Return what results can be cleaned up let result = CrucibleResources::V1(CrucibleResourcesV1 { @@ -681,7 +684,8 @@ impl DataStore { .or(dsl::volume_references.is_null()), ) .select((Dataset::as_select(), Region::as_select())) - .get_results::<(Dataset, Region)>(conn)? + .get_results_async::<(Dataset, Region)>(&conn) + .await? }, // A volume (for a disk or snapshot) may reference another nested @@ -707,11 +711,9 @@ impl DataStore { // delete a read-only downstairs running for a // snapshot that doesn't exist will return a 404, // causing the saga to error and unwind. - .filter( - dsl::snapshot_addr.eq_any( - &crucible_targets.read_only_targets, - ), - ) + .filter(dsl::snapshot_addr.eq_any( + crucible_targets.read_only_targets.clone(), + )) .filter(dsl::volume_references.eq(0)) .inner_join( dataset_dsl::dataset @@ -721,7 +723,10 @@ impl DataStore { Dataset::as_select(), RegionSnapshot::as_select(), )) - .get_results::<(Dataset, RegionSnapshot)>(conn)? + .get_results_async::<(Dataset, RegionSnapshot)>( + &conn, + ) + .await? }, }); @@ -742,7 +747,8 @@ impl DataStore { })?, ), )) - .execute(conn)?; + .execute_async(&conn) + .await?; Ok(result) }) @@ -750,10 +756,7 @@ impl DataStore { .map_err(|e| match e { TxnError::CustomError( DecreaseCrucibleResourcesError::DieselError(e), - ) => public_error_from_diesel_pool( - e.into(), - ErrorHandler::Server, - ), + ) => public_error_from_diesel(e.into(), ErrorHandler::Server), _ => { Error::internal_error(&format!("Transaction error: {}", e)) @@ -799,8 +802,9 @@ impl DataStore { // data from original volume_id. // - Put the new temp VCR into the temp volume.data, update the // temp_volume in the database. - self.pool() - .transaction(move |conn| { + self.pool_connection_unauthorized() + .await? + .transaction_async(|conn| async move { // Grab the volume in question. If the volume record was already // deleted then we can just return. let volume = { @@ -809,7 +813,8 @@ impl DataStore { let volume = dsl::volume .filter(dsl::id.eq(volume_id)) .select(Volume::as_select()) - .get_result(conn) + .get_result_async(&conn) + .await .optional()?; let volume = if let Some(v) = volume { @@ -882,7 +887,8 @@ impl DataStore { let num_updated = diesel::update(volume_dsl::volume) .filter(volume_dsl::id.eq(volume_id)) .set(volume_dsl::data.eq(new_volume_data)) - .execute(conn)?; + .execute_async(&conn) + .await?; // This should update just one row. If it does // not, then something is terribly wrong in the @@ -920,7 +926,8 @@ impl DataStore { .filter(volume_dsl::id.eq(temp_volume_id)) .filter(volume_dsl::time_deleted.is_null()) .set(volume_dsl::data.eq(rop_volume_data)) - .execute(conn)?; + .execute_async(&conn) + .await?; if num_updated != 1 { return Err(TxnError::CustomError( RemoveReadOnlyParentError::UnexpectedDatabaseUpdate(num_updated, 1), @@ -946,7 +953,7 @@ impl DataStore { .map_err(|e| match e { TxnError::CustomError( RemoveReadOnlyParentError::DieselError(e), - ) => public_error_from_diesel_pool( + ) => public_error_from_diesel( e.into(), ErrorHandler::Server, ), diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index f82270a27f..af7ea93456 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -10,8 +10,8 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::diesel_pool_result_optional; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::diesel_result_optional; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::fixed_data::vpc::SERVICES_VPC_ID; @@ -279,9 +279,9 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::project_id.eq(authz_project.id())) .select(Vpc::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn project_create_vpc( @@ -312,23 +312,22 @@ impl DataStore { let name = vpc_query.vpc.identity.name.clone(); let project_id = vpc_query.vpc.project_id; + let conn = self.pool_connection_authorized(opctx).await?; let vpc: Vpc = Project::insert_resource( project_id, diesel::insert_into(dsl::vpc).values(vpc_query), ) - .insert_and_get_result_async(self.pool()) + .insert_and_get_result_async(&conn) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { type_name: ResourceType::Project, lookup_type: LookupType::ById(project_id), }, - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Conflict(ResourceType::Vpc, name.as_str()), - ) - } + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict(ResourceType::Vpc, name.as_str()), + ), })?; Ok(( authz::Vpc::new( @@ -354,10 +353,10 @@ impl DataStore { .filter(dsl::id.eq(authz_vpc.id())) .set(updates) .returning(Vpc::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_vpc), ) @@ -390,16 +389,18 @@ impl DataStore { // but we can't have NICs be a child of both tables at this point, and // we need to prevent VPC Subnets from being deleted while they have // NICs in them as well. - if diesel_pool_result_optional( + if diesel_result_optional( vpc_subnet::dsl::vpc_subnet .filter(vpc_subnet::dsl::vpc_id.eq(authz_vpc.id())) .filter(vpc_subnet::dsl::time_deleted.is_null()) .select(vpc_subnet::dsl::id) .limit(1) - .first_async::(self.pool_authorized(opctx).await?) + .first_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await, ) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))? + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .is_some() { return Err(Error::InvalidRequest { @@ -416,10 +417,10 @@ impl DataStore { .filter(dsl::id.eq(authz_vpc.id())) .filter(dsl::subnet_gen.eq(db_vpc.subnet_gen)) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_vpc), ) @@ -448,14 +449,15 @@ impl DataStore { opctx.authorize(authz::Action::Read, authz_vpc).await?; use db::schema::vpc_firewall_rule::dsl; + let conn = self.pool_connection_authorized(opctx).await?; dsl::vpc_firewall_rule .filter(dsl::time_deleted.is_null()) .filter(dsl::vpc_id.eq(authz_vpc.id())) .order(dsl::name.asc()) .select(VpcFirewallRule::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn vpc_delete_all_firewall_rules( @@ -466,16 +468,17 @@ impl DataStore { opctx.authorize(authz::Action::Modify, authz_vpc).await?; use db::schema::vpc_firewall_rule::dsl; + let conn = self.pool_connection_authorized(opctx).await?; let now = Utc::now(); // TODO-performance: Paginate this update to avoid long queries diesel::update(dsl::vpc_firewall_rule) .filter(dsl::time_deleted.is_null()) .filter(dsl::vpc_id.eq(authz_vpc.id())) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*conn) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_vpc), ) @@ -525,7 +528,7 @@ impl DataStore { // hold a transaction open across multiple roundtrips from the database, // but for now we're using a transaction due to the severely decreased // legibility of CTEs via diesel right now. - self.pool_authorized(opctx) + self.pool_connection_authorized(opctx) .await? .transaction_async(|conn| async move { delete_old_query.execute_async(&conn).await?; @@ -553,7 +556,7 @@ impl DataStore { TxnError::CustomError( FirewallUpdateError::CollectionNotFound, ) => Error::not_found_by_id(ResourceType::Vpc, &authz_vpc.id()), - TxnError::Pool(e) => public_error_from_diesel_pool( + TxnError::Connection(e) => public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_vpc), ), @@ -604,11 +607,12 @@ impl DataStore { sleds = sleds.filter(sled::id.eq_any(sleds_filter.to_vec())); } + let conn = self.pool_connection_unauthorized().await?; sleds .intersect(instance_query.union(service_query)) - .get_results_async(self.pool()) + .get_results_async(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn vpc_subnet_list( @@ -620,6 +624,7 @@ impl DataStore { opctx.authorize(authz::Action::ListChildren, authz_vpc).await?; use db::schema::vpc_subnet::dsl; + let conn = self.pool_connection_authorized(opctx).await?; match pagparams { PaginatedBy::Id(pagparams) => { paginated(dsl::vpc_subnet, dsl::id, &pagparams) @@ -633,9 +638,9 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::vpc_id.eq(authz_vpc.id())) .select(VpcSubnet::as_select()) - .load_async(self.pool_authorized(opctx).await?) + .load_async(&*conn) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Insert a VPC Subnet, checking for unique IP address ranges. @@ -668,12 +673,17 @@ impl DataStore { ) -> Result { use db::schema::vpc_subnet::dsl; let values = FilterConflictingVpcSubnetRangesQuery::new(subnet.clone()); + let conn = self + .pool_connection_unauthorized() + .await + .map_err(SubnetError::External)?; + diesel::insert_into(dsl::vpc_subnet) .values(values) .returning(VpcSubnet::as_returning()) - .get_result_async(self.pool()) + .get_result_async(&*conn) .await - .map_err(|e| SubnetError::from_pool(e, &subnet)) + .map_err(|e| SubnetError::from_diesel(e, &subnet)) } pub async fn vpc_delete_subnet( @@ -687,17 +697,19 @@ impl DataStore { use db::schema::network_interface; use db::schema::vpc_subnet::dsl; + let conn = self.pool_connection_authorized(opctx).await?; + // Verify there are no child network interfaces in this VPC Subnet - if diesel_pool_result_optional( + if diesel_result_optional( network_interface::dsl::network_interface .filter(network_interface::dsl::subnet_id.eq(authz_subnet.id())) .filter(network_interface::dsl::time_deleted.is_null()) .select(network_interface::dsl::id) .limit(1) - .first_async::(self.pool_authorized(opctx).await?) + .first_async::(&*conn) .await, ) - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server))? + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? .is_some() { return Err(Error::InvalidRequest { @@ -715,10 +727,10 @@ impl DataStore { .filter(dsl::id.eq(authz_subnet.id())) .filter(dsl::rcgen.eq(db_subnet.rcgen)) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_subnet), ) @@ -748,10 +760,10 @@ impl DataStore { .filter(dsl::id.eq(authz_subnet.id())) .set(updates) .returning(VpcSubnet::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_subnet), ) @@ -782,10 +794,10 @@ impl DataStore { .filter(dsl::subnet_id.eq(authz_subnet.id())) .select(InstanceNetworkInterface::as_select()) .load_async::( - self.pool_authorized(opctx).await?, + &*self.pool_connection_authorized(opctx).await?, ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn vpc_router_list( @@ -810,9 +822,11 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::vpc_id.eq(authz_vpc.id())) .select(VpcRouter::as_select()) - .load_async::(self.pool_authorized(opctx).await?) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn vpc_create_router( @@ -830,10 +844,10 @@ impl DataStore { .on_conflict(dsl::id) .do_nothing() .returning(VpcRouter::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::VpcRouter, @@ -864,10 +878,10 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::id.eq(authz_router.id())) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool()) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_router), ) @@ -889,10 +903,10 @@ impl DataStore { .filter(dsl::id.eq(authz_router.id())) .set(updates) .returning(VpcRouter::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_router), ) @@ -922,10 +936,10 @@ impl DataStore { .filter(dsl::vpc_router_id.eq(authz_router.id())) .select(RouterRoute::as_select()) .load_async::( - self.pool_authorized(opctx).await?, + &*self.pool_connection_authorized(opctx).await?, ) .await - .map_err(|e| public_error_from_diesel_pool(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn router_create_route( @@ -945,22 +959,22 @@ impl DataStore { router_id, diesel::insert_into(dsl::router_route).values(route), ) - .insert_and_get_result_async(self.pool_authorized(opctx).await?) + .insert_and_get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { type_name: ResourceType::VpcRouter, lookup_type: LookupType::ById(router_id), }, - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::RouterRoute, - name.as_str(), - ), - ) - } + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::RouterRoute, + name.as_str(), + ), + ), }) } @@ -977,10 +991,10 @@ impl DataStore { .filter(dsl::time_deleted.is_null()) .filter(dsl::id.eq(authz_route.id())) .set(dsl::time_deleted.eq(now)) - .execute_async(self.pool_authorized(opctx).await?) + .execute_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_route), ) @@ -1002,10 +1016,10 @@ impl DataStore { .filter(dsl::id.eq(authz_route.id())) .set(route_update) .returning(RouterRoute::as_returning()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_route), ) @@ -1037,11 +1051,11 @@ impl DataStore { vpc_subnet::ipv4_block, vpc_subnet::ipv6_block, )) - .get_results_async::(self.pool()) + .get_results_async::( + &*self.pool_connection_unauthorized().await?, + ) .await - .map_err(|e| { - public_error_from_diesel_pool(e, ErrorHandler::Server) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; let mut result = BTreeMap::new(); for subnet in subnets { @@ -1063,10 +1077,10 @@ impl DataStore { .filter(dsl::vni.eq(vni)) .filter(dsl::time_deleted.is_null()) .select(Vpc::as_select()) - .get_result_async(self.pool_authorized(opctx).await?) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| { - public_error_from_diesel_pool( + public_error_from_diesel( e, ErrorHandler::NotFoundByLookup( ResourceType::Vpc, diff --git a/nexus/db-queries/src/db/datastore/zpool.rs b/nexus/db-queries/src/db/datastore/zpool.rs index b2fb6cdf7a..5d6c0844ef 100644 --- a/nexus/db-queries/src/db/datastore/zpool.rs +++ b/nexus/db-queries/src/db/datastore/zpool.rs @@ -8,7 +8,7 @@ use super::DataStore; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::public_error_from_diesel_pool; +use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::identity::Asset; use crate::db::model::Sled; @@ -39,22 +39,22 @@ impl DataStore { dsl::total_size.eq(excluded(dsl::total_size)), )), ) - .insert_and_get_result_async(self.pool()) + .insert_and_get_result_async( + &*self.pool_connection_unauthorized().await?, + ) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { type_name: ResourceType::Sled, lookup_type: LookupType::ById(sled_id), }, - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel_pool( - e, - ErrorHandler::Conflict( - ResourceType::Zpool, - &zpool.id().to_string(), - ), - ) - } + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Zpool, + &zpool.id().to_string(), + ), + ), }) } } diff --git a/nexus/db-queries/src/db/error.rs b/nexus/db-queries/src/db/error.rs index 59094d2e0b..f7402bb8c7 100644 --- a/nexus/db-queries/src/db/error.rs +++ b/nexus/db-queries/src/db/error.rs @@ -4,7 +4,7 @@ //! Error handling and conversions. -use async_bb8_diesel::{ConnectionError, PoolError, PoolResult}; +use async_bb8_diesel::ConnectionError; use diesel::result::DatabaseErrorInformation; use diesel::result::DatabaseErrorKind as DieselErrorKind; use diesel::result::Error as DieselError; @@ -25,23 +25,15 @@ pub enum TransactionError { /// /// This error covers failure due to accessing the DB pool or errors /// propagated from the DB itself. - #[error("Pool error: {0}")] - Pool(#[from] async_bb8_diesel::PoolError), + #[error("Connection error: {0}")] + Connection(#[from] async_bb8_diesel::ConnectionError), } // Maps a "diesel error" into a "pool error", which // is already contained within the error type. impl From for TransactionError { fn from(err: DieselError) -> Self { - Self::Pool(PoolError::Connection(ConnectionError::Query(err))) - } -} - -// Maps a "connection error" into a "pool error", which -// is already contained within the error type. -impl From for TransactionError { - fn from(err: async_bb8_diesel::ConnectionError) -> Self { - Self::Pool(PoolError::Connection(err)) + Self::Connection(ConnectionError::Query(err)) } } @@ -58,22 +50,16 @@ impl TransactionError { /// [1]: https://www.cockroachlabs.com/docs/v23.1/transaction-retry-error-reference#client-side-retry-handling pub fn retry_transaction(&self) -> bool { match &self { - TransactionError::Pool(e) => match e { - PoolError::Connection(ConnectionError::Query( - DieselError::DatabaseError(kind, boxed_error_information), - )) => match kind { - DieselErrorKind::SerializationFailure => { - return boxed_error_information - .message() - .starts_with("restart transaction"); - } - - _ => false, - }, - + TransactionError::Connection(ConnectionError::Query( + DieselError::DatabaseError(kind, boxed_error_information), + )) => match kind { + DieselErrorKind::SerializationFailure => { + return boxed_error_information + .message() + .starts_with("restart transaction"); + } _ => false, }, - _ => false, } } @@ -110,14 +96,12 @@ fn format_database_error( /// Like [`diesel::result::OptionalExtension::optional`]. This turns Ok(v) /// into Ok(Some(v)), Err("NotFound") into Ok(None), and leave all other values /// unchanged. -pub fn diesel_pool_result_optional( - result: PoolResult, -) -> PoolResult> { +pub fn diesel_result_optional( + result: Result, +) -> Result, ConnectionError> { match result { Ok(v) => Ok(Some(v)), - Err(PoolError::Connection(ConnectionError::Query( - DieselError::NotFound, - ))) => Ok(None), + Err(ConnectionError::Query(DieselError::NotFound)) => Ok(None), Err(e) => Err(e), } } @@ -153,57 +137,46 @@ pub enum ErrorHandler<'a> { Server, } -/// Converts a Diesel pool error to a public-facing error. +/// Converts a Diesel connection error to a public-facing error. /// /// [`ErrorHandler`] may be used to add additional handlers for the error /// being returned. -pub fn public_error_from_diesel_pool( - error: PoolError, +pub fn public_error_from_diesel( + error: ConnectionError, handler: ErrorHandler<'_>, ) -> PublicError { - public_error_from_diesel_pool_helper(error, |error| match handler { - ErrorHandler::NotFoundByResource(resource) => { - public_error_from_diesel_lookup( - error, - resource.resource_type(), - resource.lookup_type(), - ) - } - ErrorHandler::NotFoundByLookup(resource_type, lookup_type) => { - public_error_from_diesel_lookup(error, resource_type, &lookup_type) - } - ErrorHandler::Conflict(resource_type, object_name) => { - public_error_from_diesel_create(error, resource_type, object_name) - } - ErrorHandler::Server => PublicError::internal_error(&format!( - "unexpected database error: {:#}", + match error { + ConnectionError::Connection(error) => PublicError::unavail(&format!( + "Failed to access connection pool: {}", error )), - }) -} - -/// Handles the common cases for all pool errors (particularly around transient -/// errors while delegating the special case of -/// `PoolError::Connection(ConnectionError::Query(diesel_error))` to -/// `make_query_error(diesel_error)`, allowing the caller to decide how to -/// format a message for that case. -fn public_error_from_diesel_pool_helper( - error: PoolError, - make_query_error: F, -) -> PublicError -where - F: FnOnce(DieselError) -> PublicError, -{ - match error { - PoolError::Connection(error) => match error { - ConnectionError::Connection(error) => PublicError::unavail( - &format!("Failed to access connection pool: {}", error), - ), - ConnectionError::Query(error) => make_query_error(error), + ConnectionError::Query(error) => match handler { + ErrorHandler::NotFoundByResource(resource) => { + public_error_from_diesel_lookup( + error, + resource.resource_type(), + resource.lookup_type(), + ) + } + ErrorHandler::NotFoundByLookup(resource_type, lookup_type) => { + public_error_from_diesel_lookup( + error, + resource_type, + &lookup_type, + ) + } + ErrorHandler::Conflict(resource_type, object_name) => { + public_error_from_diesel_create( + error, + resource_type, + object_name, + ) + } + ErrorHandler::Server => PublicError::internal_error(&format!( + "unexpected database error: {:#}", + error + )), }, - PoolError::Timeout => { - PublicError::unavail("Timeout accessing connection pool") - } } } diff --git a/nexus/db-queries/src/db/explain.rs b/nexus/db-queries/src/db/explain.rs index de834eb301..fc8098b876 100644 --- a/nexus/db-queries/src/db/explain.rs +++ b/nexus/db-queries/src/db/explain.rs @@ -5,7 +5,7 @@ //! Utility allowing Diesel to EXPLAIN queries. use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionManager, PoolError}; +use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionError}; use async_trait::async_trait; use diesel::pg::Pg; use diesel::prelude::*; @@ -48,8 +48,8 @@ pub trait ExplainableAsync { /// Asynchronously issues an explain statement. async fn explain_async( self, - pool: &bb8::Pool>, - ) -> Result; + conn: &async_bb8_diesel::Connection, + ) -> Result; } #[async_trait] @@ -64,10 +64,10 @@ where { async fn explain_async( self, - pool: &bb8::Pool>, - ) -> Result { + conn: &async_bb8_diesel::Connection, + ) -> Result { Ok(ExplainStatement { query: self } - .get_results_async::(pool) + .get_results_async::(conn) .await? .join("\n")) } @@ -167,6 +167,7 @@ mod test { let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); + let conn = pool.pool().get().await.unwrap(); create_schema(&pool).await; @@ -174,7 +175,7 @@ mod test { let explanation = dsl::test_users .filter(dsl::id.eq(Uuid::nil())) .select(User::as_select()) - .explain_async(pool.pool()) + .explain_async(&conn) .await .unwrap(); @@ -190,6 +191,7 @@ mod test { let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; let pool = db::Pool::new(&logctx.log, &cfg); + let conn = pool.pool().get().await.unwrap(); create_schema(&pool).await; @@ -197,7 +199,7 @@ mod test { let explanation = dsl::test_users .filter(dsl::age.eq(2)) .select(User::as_select()) - .explain_async(pool.pool()) + .explain_async(&conn) .await .unwrap(); diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index e7e7bb47fc..72a32f562c 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -11,7 +11,7 @@ use crate::{ authz, context::OpContext, db, - db::error::{public_error_from_diesel_pool, ErrorHandler}, + db::error::{public_error_from_diesel, ErrorHandler}, }; use async_bb8_diesel::AsyncRunQueryDsl; use db_macros::lookup_resource; diff --git a/nexus/db-queries/src/db/pagination.rs b/nexus/db-queries/src/db/pagination.rs index 50da36c156..dd7daab14f 100644 --- a/nexus/db-queries/src/db/pagination.rs +++ b/nexus/db-queries/src/db/pagination.rs @@ -214,14 +214,12 @@ mod test { async fn populate_users(pool: &db::Pool, values: &Vec<(i64, i64)>) { use schema::test_users::dsl; + let conn = pool.pool().get().await.unwrap(); + // The indexes here work around the check that prevents full table // scans. - pool.pool() - .get() - .await - .unwrap() - .batch_execute_async( - "CREATE TABLE test_users ( + conn.batch_execute_async( + "CREATE TABLE test_users ( id UUID PRIMARY KEY, age INT NOT NULL, height INT NOT NULL @@ -229,9 +227,9 @@ mod test { CREATE INDEX ON test_users (age, height); CREATE INDEX ON test_users (height, age);", - ) - .await - .unwrap(); + ) + .await + .unwrap(); let users: Vec = values .iter() @@ -244,7 +242,7 @@ mod test { diesel::insert_into(dsl::test_users) .values(users) - .execute_async(pool.pool()) + .execute_async(&*conn) .await .unwrap(); } @@ -254,7 +252,8 @@ mod test { pool: &db::Pool, query: BoxedQuery, ) -> Vec { - query.select(User::as_select()).load_async(pool.pool()).await.unwrap() + let conn = pool.pool().get().await.unwrap(); + query.select(User::as_select()).load_async(&*conn).await.unwrap() } #[tokio::test] diff --git a/nexus/db-queries/src/db/queries/external_ip.rs b/nexus/db-queries/src/db/queries/external_ip.rs index e5f57181fa..18360e1045 100644 --- a/nexus/db-queries/src/db/queries/external_ip.rs +++ b/nexus/db-queries/src/db/queries/external_ip.rs @@ -42,7 +42,7 @@ const REALLOCATION_WITH_DIFFERENT_IP_SENTINEL: &'static str = "Reallocation of IP with different value"; /// Translates a generic pool error to an external error. -pub fn from_pool(e: async_bb8_diesel::PoolError) -> external::Error { +pub fn from_diesel(e: async_bb8_diesel::ConnectionError) -> external::Error { use crate::db::error; let sentinels = [REALLOCATION_WITH_DIFFERENT_IP_SENTINEL]; @@ -58,7 +58,7 @@ pub fn from_pool(e: async_bb8_diesel::PoolError) -> external::Error { } } - error::public_error_from_diesel_pool(e, error::ErrorHandler::Server) + error::public_error_from_diesel(e, error::ErrorHandler::Server) } const MAX_PORT: u16 = u16::MAX; @@ -877,15 +877,16 @@ mod tests { is_default, ); + let conn = self + .db_datastore + .pool_connection_authorized(&self.opctx) + .await + .unwrap(); + use crate::db::schema::ip_pool::dsl as ip_pool_dsl; diesel::insert_into(ip_pool_dsl::ip_pool) .values(pool.clone()) - .execute_async( - self.db_datastore - .pool_authorized(&self.opctx) - .await - .unwrap(), - ) + .execute_async(&*conn) .await .expect("Failed to create IP Pool"); @@ -895,16 +896,16 @@ mod tests { async fn initialize_ip_pool(&self, name: &str, range: IpRange) { // Find the target IP pool use crate::db::schema::ip_pool::dsl as ip_pool_dsl; + let conn = self + .db_datastore + .pool_connection_authorized(&self.opctx) + .await + .unwrap(); let pool = ip_pool_dsl::ip_pool .filter(ip_pool_dsl::name.eq(name.to_string())) .filter(ip_pool_dsl::time_deleted.is_null()) .select(IpPool::as_select()) - .get_result_async( - self.db_datastore - .pool_authorized(&self.opctx) - .await - .unwrap(), - ) + .get_result_async(&*conn) .await .expect("Failed to 'SELECT' IP Pool"); @@ -915,7 +916,11 @@ mod tests { ) .values(pool_range) .execute_async( - self.db_datastore.pool_authorized(&self.opctx).await.unwrap(), + &*self + .db_datastore + .pool_connection_authorized(&self.opctx) + .await + .unwrap(), ) .await .expect("Failed to create IP Pool range"); diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs index 5bb9da928e..877daad9e3 100644 --- a/nexus/db-queries/src/db/queries/network_interface.rs +++ b/nexus/db-queries/src/db/queries/network_interface.rs @@ -125,22 +125,21 @@ impl InsertError { /// can generate, especially the intentional errors that indicate either IP /// address exhaustion or an attempt to attach an interface to an instance /// that is already associated with another VPC. - pub fn from_pool( - e: async_bb8_diesel::PoolError, + pub fn from_diesel( + e: async_bb8_diesel::ConnectionError, interface: &IncompleteNetworkInterface, ) -> Self { use crate::db::error; use async_bb8_diesel::ConnectionError; - use async_bb8_diesel::PoolError; use diesel::result::Error; match e { // Catch the specific errors designed to communicate the failures we // want to distinguish - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(_, _), - )) => decode_database_error(e, interface), + ConnectionError::Query(Error::DatabaseError(_, _)) => { + decode_database_error(e, interface) + } // Any other error at all is a bug - _ => InsertError::External(error::public_error_from_diesel_pool( + _ => InsertError::External(error::public_error_from_diesel( e, error::ErrorHandler::Server, )), @@ -224,12 +223,11 @@ impl InsertError { /// As such, it naturally is extremely tightly coupled to the database itself, /// including the software version and our schema. fn decode_database_error( - err: async_bb8_diesel::PoolError, + err: async_bb8_diesel::ConnectionError, interface: &IncompleteNetworkInterface, ) -> InsertError { use crate::db::error; use async_bb8_diesel::ConnectionError; - use async_bb8_diesel::PoolError; use diesel::result::DatabaseErrorKind; use diesel::result::Error; @@ -294,8 +292,9 @@ fn decode_database_error( // If the address allocation subquery fails, we'll attempt to insert // NULL for the `ip` column. This checks that the non-NULL constraint on // that colum has been violated. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::NotNullViolation, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::NotNullViolation, + ref info, )) if info.message() == IP_EXHAUSTION_ERROR_MESSAGE => { InsertError::NoAvailableIpAddresses } @@ -304,16 +303,18 @@ fn decode_database_error( // `push_ensure_unique_vpc_expression` subquery, which generates a // UUID parsing error if the resource (e.g. instance) we want to attach // to is already associated with another VPC. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::Unknown, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::Unknown, + ref info, )) if info.message() == MULTIPLE_VPC_ERROR_MESSAGE => { InsertError::ResourceSpansMultipleVpcs(interface.parent_id) } // This checks the constraint on the interface slot numbers, used to // limit total number of interfaces per resource to a maximum number. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::CheckViolation, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::CheckViolation, + ref info, )) if info.message() == NO_SLOTS_AVAILABLE_ERROR_MESSAGE => { InsertError::NoSlotsAvailable } @@ -321,8 +322,9 @@ fn decode_database_error( // If the MAC allocation subquery fails, we'll attempt to insert NULL // for the `mac` column. This checks that the non-NULL constraint on // that column has been violated. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::NotNullViolation, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::NotNullViolation, + ref info, )) if info.message() == MAC_EXHAUSTION_ERROR_MESSAGE => { InsertError::NoMacAddrressesAvailable } @@ -331,8 +333,9 @@ fn decode_database_error( // `push_ensure_unique_vpc_subnet_expression` subquery, which generates // a UUID parsing error if the resource has another interface in the VPC // Subnet of the one we're trying to insert. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::Unknown, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::Unknown, + ref info, )) if info.message() == NON_UNIQUE_VPC_SUBNET_ERROR_MESSAGE => { InsertError::NonUniqueVpcSubnets } @@ -340,8 +343,9 @@ fn decode_database_error( // This catches the UUID-cast failure intentionally introduced by // `push_instance_state_verification_subquery`, which verifies that // the instance is actually stopped when running this query. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::Unknown, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::Unknown, + ref info, )) if info.message() == INSTANCE_BAD_STATE_ERROR_MESSAGE => { assert_eq!(interface.kind, NetworkInterfaceKind::Instance); InsertError::InstanceMustBeStopped(interface.parent_id) @@ -349,16 +353,18 @@ fn decode_database_error( // This catches the UUID-cast failure intentionally introduced by // `push_instance_state_verification_subquery`, which verifies that // the instance doesn't even exist when running this query. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::Unknown, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::Unknown, + ref info, )) if info.message() == NO_INSTANCE_ERROR_MESSAGE => { assert_eq!(interface.kind, NetworkInterfaceKind::Instance); InsertError::InstanceNotFound(interface.parent_id) } // This path looks specifically at constraint names. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::UniqueViolation, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::UniqueViolation, + ref info, )) => match info.constraint_name() { // Constraint violated if a user-requested IP address has // already been assigned within the same VPC Subnet. @@ -385,7 +391,7 @@ fn decode_database_error( external::ResourceType::ServiceNetworkInterface } }; - InsertError::External(error::public_error_from_diesel_pool( + InsertError::External(error::public_error_from_diesel( err, error::ErrorHandler::Conflict( resource_type, @@ -402,14 +408,14 @@ fn decode_database_error( ) } // Any other constraint violation is a bug - _ => InsertError::External(error::public_error_from_diesel_pool( + _ => InsertError::External(error::public_error_from_diesel( err, error::ErrorHandler::Server, )), }, // Any other error at all is a bug - _ => InsertError::External(error::public_error_from_diesel_pool( + _ => InsertError::External(error::public_error_from_diesel( err, error::ErrorHandler::Server, )), @@ -1544,25 +1550,24 @@ impl DeleteError { /// can generate, specifically the intentional errors that indicate that /// either the instance is still running, or that the instance has one or /// more secondary interfaces. - pub fn from_pool( - e: async_bb8_diesel::PoolError, + pub fn from_diesel( + e: async_bb8_diesel::ConnectionError, query: &DeleteQuery, ) -> Self { use crate::db::error; use async_bb8_diesel::ConnectionError; - use async_bb8_diesel::PoolError; use diesel::result::Error; match e { // Catch the specific errors designed to communicate the failures we // want to distinguish - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(_, _), - )) => decode_delete_network_interface_database_error( - e, - query.parent_id, - ), + ConnectionError::Query(Error::DatabaseError(_, _)) => { + decode_delete_network_interface_database_error( + e, + query.parent_id, + ) + } // Any other error at all is a bug - _ => DeleteError::External(error::public_error_from_diesel_pool( + _ => DeleteError::External(error::public_error_from_diesel( e, error::ErrorHandler::Server, )), @@ -1603,12 +1608,11 @@ impl DeleteError { /// As such, it naturally is extremely tightly coupled to the database itself, /// including the software version and our schema. fn decode_delete_network_interface_database_error( - err: async_bb8_diesel::PoolError, + err: async_bb8_diesel::ConnectionError, parent_id: Uuid, ) -> DeleteError { use crate::db::error; use async_bb8_diesel::ConnectionError; - use async_bb8_diesel::PoolError; use diesel::result::DatabaseErrorKind; use diesel::result::Error; @@ -1623,8 +1627,9 @@ fn decode_delete_network_interface_database_error( // first CTE, which generates a UUID parsing error if we're trying to // delete the primary interface, and the instance also has one or more // secondaries. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::Unknown, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::Unknown, + ref info, )) if info.message() == HAS_SECONDARIES_ERROR_MESSAGE => { DeleteError::SecondariesExist(parent_id) } @@ -1632,22 +1637,24 @@ fn decode_delete_network_interface_database_error( // This catches the UUID-cast failure intentionally introduced by // `push_instance_state_verification_subquery`, which verifies that // the instance can be worked on when running this query. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::Unknown, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::Unknown, + ref info, )) if info.message() == INSTANCE_BAD_STATE_ERROR_MESSAGE => { DeleteError::InstanceBadState(parent_id) } // This catches the UUID-cast failure intentionally introduced by // `push_instance_state_verification_subquery`, which verifies that // the instance doesn't even exist when running this query. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::Unknown, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::Unknown, + ref info, )) if info.message() == NO_INSTANCE_ERROR_MESSAGE => { DeleteError::InstanceNotFound(parent_id) } // Any other error at all is a bug - _ => DeleteError::External(error::public_error_from_diesel_pool( + _ => DeleteError::External(error::public_error_from_diesel( err, error::ErrorHandler::Server, )), @@ -1883,16 +1890,18 @@ mod tests { db_datastore.project_create(&opctx, project).await.unwrap(); use crate::db::schema::vpc_subnet::dsl::vpc_subnet; - let p = db_datastore.pool_authorized(&opctx).await.unwrap(); + let conn = + db_datastore.pool_connection_authorized(&opctx).await.unwrap(); let net1 = Network::new(n_subnets); let net2 = Network::new(n_subnets); for subnet in net1.subnets.iter().chain(net2.subnets.iter()) { diesel::insert_into(vpc_subnet) .values(subnet.clone()) - .execute_async(p) + .execute_async(&*conn) .await .unwrap(); } + drop(conn); Self { logctx, opctx, diff --git a/nexus/db-queries/src/db/queries/next_item.rs b/nexus/db-queries/src/db/queries/next_item.rs index 3ba09788a0..007aec943d 100644 --- a/nexus/db-queries/src/db/queries/next_item.rs +++ b/nexus/db-queries/src/db/queries/next_item.rs @@ -593,6 +593,7 @@ mod tests { let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg)); + let conn = pool.pool().get().await.unwrap(); // We're going to operate on a separate table, for simplicity. setup_test_schema(&pool).await; @@ -607,7 +608,7 @@ mod tests { let it = diesel::insert_into(item::dsl::item) .values(query) .returning(Item::as_returning()) - .get_result_async(pool.pool()) + .get_result_async(&*conn) .await .unwrap(); assert_eq!(it.value, 0); @@ -616,7 +617,7 @@ mod tests { let it = diesel::insert_into(item::dsl::item) .values(query) .returning(Item::as_returning()) - .get_result_async(pool.pool()) + .get_result_async(&*conn) .await .unwrap(); assert_eq!(it.value, 1); @@ -628,7 +629,7 @@ mod tests { let it = diesel::insert_into(item::dsl::item) .values(query) .returning(Item::as_returning()) - .get_result_async(pool.pool()) + .get_result_async(&*conn) .await .unwrap(); assert_eq!(it.value, 10); @@ -638,7 +639,7 @@ mod tests { let it = diesel::insert_into(item::dsl::item) .values(query) .returning(Item::as_returning()) - .get_result_async(pool.pool()) + .get_result_async(&*conn) .await .unwrap(); assert_eq!(it.value, 2); diff --git a/nexus/db-queries/src/db/queries/region_allocation.rs b/nexus/db-queries/src/db/queries/region_allocation.rs index 674a525c5c..b071ee3f44 100644 --- a/nexus/db-queries/src/db/queries/region_allocation.rs +++ b/nexus/db-queries/src/db/queries/region_allocation.rs @@ -36,7 +36,7 @@ const NOT_ENOUGH_UNIQUE_ZPOOLS_SENTINEL: &'static str = /// Translates a generic pool error to an external error based /// on messages which may be emitted during region provisioning. -pub fn from_pool(e: async_bb8_diesel::PoolError) -> external::Error { +pub fn from_diesel(e: async_bb8_diesel::ConnectionError) -> external::Error { use crate::db::error; let sentinels = [ @@ -66,7 +66,7 @@ pub fn from_pool(e: async_bb8_diesel::PoolError) -> external::Error { } } - error::public_error_from_diesel_pool(e, error::ErrorHandler::Server) + error::public_error_from_diesel(e, error::ErrorHandler::Server) } /// A subquery to find all old regions associated with a particular volume. diff --git a/nexus/db-queries/src/db/queries/vpc_subnet.rs b/nexus/db-queries/src/db/queries/vpc_subnet.rs index 78da549620..bbb229da1e 100644 --- a/nexus/db-queries/src/db/queries/vpc_subnet.rs +++ b/nexus/db-queries/src/db/queries/vpc_subnet.rs @@ -28,13 +28,12 @@ pub enum SubnetError { impl SubnetError { /// Construct a `SubnetError` from a Diesel error, catching the desired /// cases and building useful errors. - pub fn from_pool( - e: async_bb8_diesel::PoolError, + pub fn from_diesel( + e: async_bb8_diesel::ConnectionError, subnet: &VpcSubnet, ) -> Self { use crate::db::error; use async_bb8_diesel::ConnectionError; - use async_bb8_diesel::PoolError; use diesel::result::DatabaseErrorKind; use diesel::result::Error; const IPV4_OVERLAP_ERROR_MESSAGE: &str = @@ -44,33 +43,27 @@ impl SubnetError { const NAME_CONFLICT_CONSTRAINT: &str = "vpc_subnet_vpc_id_name_key"; match e { // Attempt to insert overlapping IPv4 subnet - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError( - DatabaseErrorKind::NotNullViolation, - ref info, - ), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::NotNullViolation, + ref info, )) if info.message() == IPV4_OVERLAP_ERROR_MESSAGE => { SubnetError::OverlappingIpRange(subnet.ipv4_block.0 .0.into()) } // Attempt to insert overlapping IPv6 subnet - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError( - DatabaseErrorKind::NotNullViolation, - ref info, - ), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::NotNullViolation, + ref info, )) if info.message() == IPV6_OVERLAP_ERROR_MESSAGE => { SubnetError::OverlappingIpRange(subnet.ipv6_block.0 .0.into()) } // Conflicting name for the subnet within a VPC - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError( - DatabaseErrorKind::UniqueViolation, - ref info, - ), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::UniqueViolation, + ref info, )) if info.constraint_name() == Some(NAME_CONFLICT_CONSTRAINT) => { - SubnetError::External(error::public_error_from_diesel_pool( + SubnetError::External(error::public_error_from_diesel( e, error::ErrorHandler::Conflict( external::ResourceType::VpcSubnet, @@ -80,7 +73,7 @@ impl SubnetError { } // Any other error at all is a bug - _ => SubnetError::External(error::public_error_from_diesel_pool( + _ => SubnetError::External(error::public_error_from_diesel( e, error::ErrorHandler::Server, )), diff --git a/nexus/db-queries/src/db/true_or_cast_error.rs b/nexus/db-queries/src/db/true_or_cast_error.rs index 6f14cd4642..e04d865182 100644 --- a/nexus/db-queries/src/db/true_or_cast_error.rs +++ b/nexus/db-queries/src/db/true_or_cast_error.rs @@ -77,11 +77,10 @@ where /// Returns one of the sentinels if it matches the expected value from /// a [`TrueOrCastError`]. pub fn matches_sentinel( - e: &async_bb8_diesel::PoolError, + e: &async_bb8_diesel::ConnectionError, sentinels: &[&'static str], ) -> Option<&'static str> { use async_bb8_diesel::ConnectionError; - use async_bb8_diesel::PoolError; use diesel::result::DatabaseErrorKind; use diesel::result::Error; @@ -94,8 +93,9 @@ pub fn matches_sentinel( match e { // Catch the specific errors designed to communicate the failures we // want to distinguish. - PoolError::Connection(ConnectionError::Query( - Error::DatabaseError(DatabaseErrorKind::Unknown, ref info), + ConnectionError::Query(Error::DatabaseError( + DatabaseErrorKind::Unknown, + ref info, )) => { for sentinel in sentinels { if info.message() == bool_parse_error(sentinel) { diff --git a/nexus/db-queries/src/db/update_and_check.rs b/nexus/db-queries/src/db/update_and_check.rs index 8c7845b61b..96cb3e4c79 100644 --- a/nexus/db-queries/src/db/update_and_check.rs +++ b/nexus/db-queries/src/db/update_and_check.rs @@ -5,7 +5,7 @@ //! CTE implementation for "UPDATE with extended return status". use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, PoolError}; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::associations::HasTable; use diesel::pg::Pg; use diesel::prelude::*; @@ -153,16 +153,13 @@ where /// - Ok(Row exists and was updated) /// - Ok(Row exists, but was not updated) /// - Error (row doesn't exist, or other diesel error) - pub async fn execute_and_check( + pub async fn execute_and_check( self, - conn: &(impl async_bb8_diesel::AsyncConnection - + Sync), - ) -> Result, PoolError> + conn: &async_bb8_diesel::Connection, + ) -> Result, async_bb8_diesel::ConnectionError> where // We require this bound to ensure that "Self" is runnable as query. Self: LoadQuery<'static, DbConnection, (Option, Option, Q)>, - ConnErr: From + Send + 'static, - PoolError: From, { let (id0, id1, found) = self.get_result_async::<(Option, Option, Q)>(conn).await?; diff --git a/nexus/src/app/background/dns_config.rs b/nexus/src/app/background/dns_config.rs index c0aaa267a2..654e9c0bf1 100644 --- a/nexus/src/app/background/dns_config.rs +++ b/nexus/src/app/background/dns_config.rs @@ -220,7 +220,9 @@ mod test { { use nexus_db_queries::db::schema::dns_version::dsl; diesel::delete(dsl::dns_version.filter(dsl::version.eq(2))) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap(); } @@ -236,7 +238,7 @@ mod test { // Similarly, wipe all of the state and verify that we handle that okay. datastore - .pool_for_tests() + .pool_connection_for_tests() .await .unwrap() .transaction_async(|conn| async move { diff --git a/nexus/src/app/background/dns_servers.rs b/nexus/src/app/background/dns_servers.rs index 419b94d360..3a75c09302 100644 --- a/nexus/src/app/background/dns_servers.rs +++ b/nexus/src/app/background/dns_servers.rs @@ -237,7 +237,9 @@ mod test { SocketAddrV6::new(Ipv6Addr::LOCALHOST, 1, 0, 0), ServiceKind::InternalDns, )) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap(); } @@ -265,7 +267,9 @@ mod test { diesel::insert_into(dsl::service) .values(new_services) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap(); } @@ -281,7 +285,9 @@ mod test { diesel::delete( dsl::service.filter(dsl::kind.eq(ServiceKind::InternalDns)), ) - .execute_async(datastore.pool_for_tests().await.unwrap()) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap(); } diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 5d1568bcb5..aa949bbc9f 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -370,7 +370,7 @@ pub mod test { ) { type TxnError = TransactionError<()>; { - let conn = datastore.pool_for_tests().await.unwrap(); + let conn = datastore.pool_connection_for_tests().await.unwrap(); let _: Result<(), TxnError> = conn .transaction_async(|conn| async move { { diff --git a/nexus/src/app/sagas/disk_create.rs b/nexus/src/app/sagas/disk_create.rs index 8e2e1d0a04..cca36cefa7 100644 --- a/nexus/src/app/sagas/disk_create.rs +++ b/nexus/src/app/sagas/disk_create.rs @@ -921,7 +921,9 @@ pub(crate) mod test { dsl::disk .filter(dsl::time_deleted.is_null()) .select(Disk::as_select()) - .first_async::(datastore.pool_for_tests().await.unwrap()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .optional() .unwrap() @@ -935,7 +937,9 @@ pub(crate) mod test { dsl::volume .filter(dsl::time_deleted.is_null()) .select(Volume::as_select()) - .first_async::(datastore.pool_for_tests().await.unwrap()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .optional() .unwrap() @@ -951,7 +955,7 @@ pub(crate) mod test { dsl::virtual_provisioning_resource .select(VirtualProvisioningResource::as_select()) .first_async::( - datastore.pool_for_tests().await.unwrap(), + &*datastore.pool_connection_for_tests().await.unwrap(), ) .await .optional() @@ -966,7 +970,7 @@ pub(crate) mod test { use nexus_db_queries::db::schema::virtual_provisioning_collection::dsl; datastore - .pool_for_tests() + .pool_connection_for_tests() .await .unwrap() .transaction_async(|conn| async move { diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index d5af080381..6fc93ce8db 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -1467,7 +1467,9 @@ pub mod test { dsl::instance .filter(dsl::time_deleted.is_null()) .select(Instance::as_select()) - .first_async::(datastore.pool_for_tests().await.unwrap()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .optional() .unwrap() @@ -1484,7 +1486,7 @@ pub mod test { .filter(dsl::kind.eq(NetworkInterfaceKind::Instance)) .select(NetworkInterface::as_select()) .first_async::( - datastore.pool_for_tests().await.unwrap(), + &*datastore.pool_connection_for_tests().await.unwrap(), ) .await .optional() @@ -1501,7 +1503,7 @@ pub mod test { .filter(dsl::is_service.eq(false)) .select(ExternalIp::as_select()) .first_async::( - datastore.pool_for_tests().await.unwrap(), + &*datastore.pool_connection_for_tests().await.unwrap(), ) .await .optional() @@ -1516,7 +1518,7 @@ pub mod test { use nexus_db_queries::db::schema::sled_resource::dsl; datastore - .pool_for_tests() + .pool_connection_for_tests() .await .unwrap() .transaction_async(|conn| async move { @@ -1550,7 +1552,7 @@ pub mod test { use nexus_db_queries::db::model::VirtualProvisioningResource; use nexus_db_queries::db::schema::virtual_provisioning_resource::dsl; - datastore.pool_for_tests() + datastore.pool_connection_for_tests() .await .unwrap() .transaction_async(|conn| async move { @@ -1578,7 +1580,7 @@ pub mod test { use nexus_db_queries::db::schema::virtual_provisioning_collection::dsl; datastore - .pool_for_tests() + .pool_connection_for_tests() .await .unwrap() .transaction_async(|conn| async move { @@ -1615,7 +1617,9 @@ pub mod test { .filter(dsl::time_deleted.is_null()) .filter(dsl::name.eq(DISK_NAME)) .select(Disk::as_select()) - .first_async::(datastore.pool_for_tests().await.unwrap()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .unwrap() .runtime_state diff --git a/nexus/src/app/sagas/project_create.rs b/nexus/src/app/sagas/project_create.rs index 65efabd8e9..1cbf9070ee 100644 --- a/nexus/src/app/sagas/project_create.rs +++ b/nexus/src/app/sagas/project_create.rs @@ -213,7 +213,9 @@ mod test { // ignore built-in services project .filter(dsl::id.ne(*SERVICES_PROJECT_ID)) .select(Project::as_select()) - .first_async::(datastore.pool_for_tests().await.unwrap()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .optional() .unwrap() @@ -230,7 +232,7 @@ mod test { use nexus_db_queries::db::model::VirtualProvisioningCollection; use nexus_db_queries::db::schema::virtual_provisioning_collection::dsl; - datastore.pool_for_tests() + datastore.pool_connection_for_tests() .await .unwrap() .transaction_async(|conn| async move { diff --git a/nexus/src/app/sagas/snapshot_create.rs b/nexus/src/app/sagas/snapshot_create.rs index bcebd17021..b27f4a3a9b 100644 --- a/nexus/src/app/sagas/snapshot_create.rs +++ b/nexus/src/app/sagas/snapshot_create.rs @@ -1885,7 +1885,9 @@ mod test { dsl::snapshot .filter(dsl::time_deleted.is_null()) .select(Snapshot::as_select()) - .first_async::(datastore.pool_for_tests().await.unwrap()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .optional() .unwrap() @@ -1899,7 +1901,7 @@ mod test { dsl::region_snapshot .select(RegionSnapshot::as_select()) .first_async::( - datastore.pool_for_tests().await.unwrap(), + &*datastore.pool_connection_for_tests().await.unwrap(), ) .await .optional() diff --git a/nexus/src/app/sagas/test_helpers.rs b/nexus/src/app/sagas/test_helpers.rs index 29f743a350..aa9334b682 100644 --- a/nexus/src/app/sagas/test_helpers.rs +++ b/nexus/src/app/sagas/test_helpers.rs @@ -398,7 +398,7 @@ pub(crate) async fn assert_no_failed_undo_steps( use nexus_db_queries::db::model::saga_types::SagaNodeEvent; let saga_node_events: Vec = datastore - .pool_for_tests() + .pool_connection_for_tests() .await .unwrap() .transaction_async(|conn| async move { diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index 97961a6fa1..85eed6616d 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -599,7 +599,9 @@ pub(crate) mod test { // ignore built-in services VPC .filter(dsl::id.ne(*SERVICES_VPC_ID)) .select(Vpc::as_select()) - .first_async::(datastore.pool_for_tests().await.unwrap()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .optional() .unwrap() @@ -618,7 +620,9 @@ pub(crate) mod test { // ignore built-in services VPC .filter(dsl::vpc_id.ne(*SERVICES_VPC_ID)) .select(VpcRouter::as_select()) - .first_async::(datastore.pool_for_tests().await.unwrap()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .optional() .unwrap() @@ -646,7 +650,7 @@ pub(crate) mod test { ) .select(RouterRoute::as_select()) .first_async::( - datastore.pool_for_tests().await.unwrap(), + &*datastore.pool_connection_for_tests().await.unwrap(), ) .await .optional() @@ -666,7 +670,9 @@ pub(crate) mod test { // ignore built-in services VPC .filter(dsl::vpc_id.ne(*SERVICES_VPC_ID)) .select(VpcSubnet::as_select()) - .first_async::(datastore.pool_for_tests().await.unwrap()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) .await .optional() .unwrap() @@ -686,7 +692,7 @@ pub(crate) mod test { .filter(dsl::vpc_id.ne(*SERVICES_VPC_ID)) .select(VpcFirewallRule::as_select()) .first_async::( - datastore.pool_for_tests().await.unwrap(), + &*datastore.pool_connection_for_tests().await.unwrap(), ) .await .optional() From 8a11c709b49eed5bbe980d2f3e69ddf115278914 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 29 Sep 2023 14:28:35 -0700 Subject: [PATCH 06/89] [schema] Add more strict CRDB index comparison (#4152) Adds more schema comparison for DB schemas, and adds a couple regression tests for the specific index mismatch. Fixes #4143 --- nexus/tests/integration_tests/schema.rs | 174 +++++++++++++++++++++++- 1 file changed, 172 insertions(+), 2 deletions(-) diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 67dfa6c255..49abf67cc8 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -15,7 +15,7 @@ use omicron_common::api::internal::shared::SwitchLocation; use omicron_common::nexus_config::Config; use omicron_common::nexus_config::SchemaConfig; use omicron_test_utils::dev::db::CockroachInstance; -use pretty_assertions::assert_eq; +use pretty_assertions::{assert_eq, assert_ne}; use similar_asserts; use slog::Logger; use std::collections::{BTreeMap, BTreeSet}; @@ -132,6 +132,7 @@ enum AnySqlType { String(String), Bool(bool), Uuid(Uuid), + Int8(i64), // TODO: This isn't exhaustive, feel free to add more. // // These should only be necessary for rows where the database schema changes also choose to @@ -167,6 +168,9 @@ impl<'a> tokio_postgres::types::FromSql<'a> for AnySqlType { if Uuid::accepts(ty) { return Ok(AnySqlType::Uuid(Uuid::from_sql(ty, raw)?)); } + if i64::accepts(ty) { + return Ok(AnySqlType::Int8(i64::from_sql(ty, raw)?)); + } Err(anyhow::anyhow!( "Cannot parse type {ty}. If you're trying to use this type in a table which is populated \ during a schema migration, consider adding it to `AnySqlType`." @@ -432,6 +436,16 @@ const CHECK_CONSTRAINTS: [&'static str; 4] = [ "check_clause", ]; +const CONSTRAINT_COLUMN_USAGE: [&'static str; 7] = [ + "table_catalog", + "table_schema", + "table_name", + "column_name", + "constraint_catalog", + "constraint_schema", + "constraint_name", +]; + const KEY_COLUMN_USAGE: [&'static str; 7] = [ "constraint_catalog", "constraint_schema", @@ -456,29 +470,50 @@ const REFERENTIAL_CONSTRAINTS: [&'static str; 8] = [ const VIEWS: [&'static str; 4] = ["table_catalog", "table_schema", "table_name", "view_definition"]; -const STATISTICS: [&'static str; 8] = [ +const STATISTICS: [&'static str; 11] = [ "table_catalog", "table_schema", "table_name", "non_unique", "index_schema", "index_name", + "seq_in_index", "column_name", "direction", + "storing", + "implicit", ]; +const PG_INDEXES: [&'static str; 5] = + ["schemaname", "tablename", "indexname", "tablespace", "indexdef"]; + const TABLES: [&'static str; 4] = ["table_catalog", "table_schema", "table_name", "table_type"]; +const TABLE_CONSTRAINTS: [&'static str; 9] = [ + "constraint_catalog", + "constraint_schema", + "constraint_name", + "table_catalog", + "table_schema", + "table_name", + "constraint_type", + "is_deferrable", + "initially_deferred", +]; + #[derive(Eq, PartialEq, Debug)] struct InformationSchema { columns: Vec, check_constraints: Vec, + constraint_column_usage: Vec, key_column_usage: Vec, referential_constraints: Vec, views: Vec, statistics: Vec, + pg_indexes: Vec, tables: Vec, + table_constraints: Vec, } impl InformationSchema { @@ -490,6 +525,10 @@ impl InformationSchema { self.check_constraints, other.check_constraints ); + similar_asserts::assert_eq!( + self.constraint_column_usage, + other.constraint_column_usage + ); similar_asserts::assert_eq!( self.key_column_usage, other.key_column_usage @@ -500,7 +539,12 @@ impl InformationSchema { ); similar_asserts::assert_eq!(self.views, other.views); similar_asserts::assert_eq!(self.statistics, other.statistics); + similar_asserts::assert_eq!(self.pg_indexes, other.pg_indexes); similar_asserts::assert_eq!(self.tables, other.tables); + similar_asserts::assert_eq!( + self.table_constraints, + other.table_constraints + ); } async fn new(crdb: &CockroachInstance) -> Self { @@ -524,6 +568,14 @@ impl InformationSchema { ) .await; + let constraint_column_usage = query_crdb_for_rows_of_strings( + crdb, + CONSTRAINT_COLUMN_USAGE.as_slice().into(), + "information_schema.constraint_column_usage", + None, + ) + .await; + let key_column_usage = query_crdb_for_rows_of_strings( crdb, KEY_COLUMN_USAGE.as_slice().into(), @@ -556,6 +608,14 @@ impl InformationSchema { ) .await; + let pg_indexes = query_crdb_for_rows_of_strings( + crdb, + PG_INDEXES.as_slice().into(), + "pg_indexes", + Some("schemaname = 'public'"), + ) + .await; + let tables = query_crdb_for_rows_of_strings( crdb, TABLES.as_slice().into(), @@ -564,14 +624,25 @@ impl InformationSchema { ) .await; + let table_constraints = query_crdb_for_rows_of_strings( + crdb, + TABLE_CONSTRAINTS.as_slice().into(), + "information_schema.table_constraints", + Some("table_schema = 'public'"), + ) + .await; + Self { columns, check_constraints, + constraint_column_usage, key_column_usage, referential_constraints, views, statistics, + pg_indexes, tables, + table_constraints, } } @@ -659,3 +730,102 @@ async fn dbinit_equals_sum_of_all_up() { crdb.cleanup().await.unwrap(); logctx.cleanup_successful(); } + +// Returns the InformationSchema object for a database populated via `sql`. +async fn get_information_schema(log: &Logger, sql: &str) -> InformationSchema { + let populate = false; + let mut crdb = test_setup_just_crdb(&log, populate).await; + + let client = crdb.connect().await.expect("failed to connect"); + client.batch_execute(sql).await.expect("failed to apply SQL"); + + let observed_schema = InformationSchema::new(&crdb).await; + crdb.cleanup().await.unwrap(); + observed_schema +} + +// Reproduction case for https://github.com/oxidecomputer/omicron/issues/4143 +#[tokio::test] +async fn compare_index_creation_differing_where_clause() { + let config = load_test_config(); + let logctx = LogContext::new( + "compare_index_creation_differing_where_clause", + &config.pkg.log, + ); + let log = &logctx.log; + + let schema1 = get_information_schema(log, " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.animal ( + id UUID PRIMARY KEY, + name TEXT, + time_deleted TIMESTAMPTZ + ); + + CREATE INDEX IF NOT EXISTS lookup_animal_by_name ON omicron.public.animal ( + name, id + ) WHERE name IS NOT NULL AND time_deleted IS NULL; + ").await; + + let schema2 = get_information_schema(log, " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.animal ( + id UUID PRIMARY KEY, + name TEXT, + time_deleted TIMESTAMPTZ + ); + + CREATE INDEX IF NOT EXISTS lookup_animal_by_name ON omicron.public.animal ( + name, id + ) WHERE time_deleted IS NULL; + ").await; + + // pg_indexes includes a column "indexdef" that compares partial indexes. + // This should catch the differing "WHERE" clause. + assert_ne!(schema1.pg_indexes, schema2.pg_indexes); + + logctx.cleanup_successful(); +} + +// Reproduction case for https://github.com/oxidecomputer/omicron/issues/4143 +#[tokio::test] +async fn compare_index_creation_differing_columns() { + let config = load_test_config(); + let logctx = LogContext::new( + "compare_index_creation_differing_columns", + &config.pkg.log, + ); + let log = &logctx.log; + + let schema1 = get_information_schema(log, " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.animal ( + id UUID PRIMARY KEY, + name TEXT, + time_deleted TIMESTAMPTZ + ); + + CREATE INDEX IF NOT EXISTS lookup_animal_by_name ON omicron.public.animal ( + name + ) WHERE name IS NOT NULL AND time_deleted IS NULL; + ").await; + + let schema2 = get_information_schema(log, " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.animal ( + id UUID PRIMARY KEY, + name TEXT, + time_deleted TIMESTAMPTZ + ); + + CREATE INDEX IF NOT EXISTS lookup_animal_by_name ON omicron.public.animal ( + name, id + ) WHERE name IS NOT NULL AND time_deleted IS NULL; + ").await; + + // "statistics" identifies table indices. + // These tables should differ in the "implicit" column. + assert_ne!(schema1.statistics, schema2.statistics); + + logctx.cleanup_successful(); +} From d18ad53df61334dfe83017278c5326b0c810ebda Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Mon, 2 Oct 2023 06:25:05 -0700 Subject: [PATCH 07/89] Update for RoT staging/dev 1.0.2 (#4167) --- .github/buildomat/jobs/tuf-repo.sh | 2 +- tools/dvt_dock_version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index a06468c6b2..fab6770564 100644 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -218,7 +218,7 @@ EOF done } # usage: SERIES ROT_DIR ROT_VERSION BOARDS... -add_hubris_artifacts rot-staging-dev staging/dev cert-staging-dev-v1.0.0 "${ALL_BOARDS[@]}" +add_hubris_artifacts rot-staging-dev staging/dev cert-staging-dev-v1.0.2 "${ALL_BOARDS[@]}" add_hubris_artifacts rot-prod-rel prod/rel cert-prod-rel-v1.0.0 "${ALL_BOARDS[@]}" for series in "${SERIES_LIST[@]}"; do diff --git a/tools/dvt_dock_version b/tools/dvt_dock_version index d7c2d31948..e2151b846f 100644 --- a/tools/dvt_dock_version +++ b/tools/dvt_dock_version @@ -1 +1 @@ -COMMIT=3cc151e62af190062780389eeae78937c3041021 +COMMIT=65f1979c1d3f4d0874a64144941cc41b46a70c80 From 76464a18f12a321ad2f0f40d597057e168d3ce50 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 2 Oct 2023 09:17:51 -0700 Subject: [PATCH 08/89] [schema] More schema comparisons (sequences, views, constraints) (#4153) Builds on https://github.com/oxidecomputer/omicron/pull/4152 , and compares more of the CRDB schema. This PR: - Compares sequences (even though we don't have any in Omicron yet) and adds a test for them - Adds a test to compare views - Adds a test to compare constraints --- nexus/tests/integration_tests/schema.rs | 153 ++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 49abf67cc8..2c62f156e1 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -484,6 +484,21 @@ const STATISTICS: [&'static str; 11] = [ "implicit", ]; +const SEQUENCES: [&'static str; 12] = [ + "sequence_catalog", + "sequence_schema", + "sequence_name", + "data_type", + "numeric_precision", + "numeric_precision_radix", + "numeric_scale", + "start_value", + "minimum_value", + "maximum_value", + "increment", + "cycle_option", +]; + const PG_INDEXES: [&'static str; 5] = ["schemaname", "tablename", "indexname", "tablespace", "indexdef"]; @@ -511,6 +526,7 @@ struct InformationSchema { referential_constraints: Vec, views: Vec, statistics: Vec, + sequences: Vec, pg_indexes: Vec, tables: Vec, table_constraints: Vec, @@ -539,6 +555,7 @@ impl InformationSchema { ); similar_asserts::assert_eq!(self.views, other.views); similar_asserts::assert_eq!(self.statistics, other.statistics); + similar_asserts::assert_eq!(self.sequences, other.sequences); similar_asserts::assert_eq!(self.pg_indexes, other.pg_indexes); similar_asserts::assert_eq!(self.tables, other.tables); similar_asserts::assert_eq!( @@ -608,6 +625,14 @@ impl InformationSchema { ) .await; + let sequences = query_crdb_for_rows_of_strings( + crdb, + SEQUENCES.as_slice().into(), + "information_schema.sequences", + None, + ) + .await; + let pg_indexes = query_crdb_for_rows_of_strings( crdb, PG_INDEXES.as_slice().into(), @@ -640,6 +665,7 @@ impl InformationSchema { referential_constraints, views, statistics, + sequences, pg_indexes, tables, table_constraints, @@ -829,3 +855,130 @@ async fn compare_index_creation_differing_columns() { logctx.cleanup_successful(); } + +#[tokio::test] +async fn compare_view_differing_where_clause() { + let config = load_test_config(); + let logctx = + LogContext::new("compare_view_differing_where_clause", &config.pkg.log); + let log = &logctx.log; + + let schema1 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.animal ( + id UUID PRIMARY KEY, + name TEXT, + time_deleted TIMESTAMPTZ + ); + + CREATE VIEW live_view AS + SELECT animal.id, animal.name + FROM omicron.public.animal + WHERE animal.time_deleted IS NOT NULL; + ", + ) + .await; + + let schema2 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.animal ( + id UUID PRIMARY KEY, + name TEXT, + time_deleted TIMESTAMPTZ + ); + + CREATE VIEW live_view AS + SELECT animal.id, animal.name + FROM omicron.public.animal + WHERE animal.time_deleted IS NOT NULL AND animal.name = 'Thomas'; + ", + ) + .await; + + assert_ne!(schema1.views, schema2.views); + + logctx.cleanup_successful(); +} + +#[tokio::test] +async fn compare_sequence_differing_increment() { + let config = load_test_config(); + let logctx = LogContext::new( + "compare_sequence_differing_increment", + &config.pkg.log, + ); + let log = &logctx.log; + + let schema1 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE SEQUENCE omicron.public.myseq START 1 INCREMENT 1; + ", + ) + .await; + + let schema2 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE SEQUENCE omicron.public.myseq START 1 INCREMENT 2; + ", + ) + .await; + + assert_ne!(schema1.sequences, schema2.sequences); + + logctx.cleanup_successful(); +} + +#[tokio::test] +async fn compare_table_differing_constraint() { + let config = load_test_config(); + let logctx = + LogContext::new("compare_table_differing_constraint", &config.pkg.log); + let log = &logctx.log; + + let schema1 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.animal ( + id UUID PRIMARY KEY, + name TEXT, + time_deleted TIMESTAMPTZ, + + CONSTRAINT dead_animals_have_names CHECK ( + (time_deleted IS NULL) OR + (name IS NOT NULL) + ) + ); + ", + ) + .await; + + let schema2 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.animal ( + id UUID PRIMARY KEY, + name TEXT, + time_deleted TIMESTAMPTZ, + + CONSTRAINT dead_animals_have_names CHECK ( + (time_deleted IS NULL) OR + (name IS NULL) + ) + ); + ", + ) + .await; + + assert_ne!(schema1.check_constraints, schema2.check_constraints); + logctx.cleanup_successful(); +} From 777277d073d92333f94deee3da68149705c18a78 Mon Sep 17 00:00:00 2001 From: Adam Leventhal Date: Mon, 2 Oct 2023 10:26:11 -0700 Subject: [PATCH 09/89] we don't need workspace deps for dev-tools (and omdb is stale) (#4170) We don't seem to need the `omicron-dev` dependency in `Cargo.toml` and the `omdb` one refers to its old location. --- Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0e194394f9..63d8e0b2d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -236,10 +236,8 @@ nexus-types = { path = "nexus/types" } num-integer = "0.1.45" num = { version = "0.4.1", default-features = false, features = [ "libm" ] } omicron-common = { path = "common" } -omicron-dev = { path = "dev-tools/omicron-dev" } omicron-gateway = { path = "gateway" } omicron-nexus = { path = "nexus" } -omicron-omdb = { path = "omdb" } omicron-package = { path = "package" } omicron-rpaths = { path = "rpaths" } omicron-sled-agent = { path = "sled-agent" } From 1c0553a682f21448755c700548107da4367d2443 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 2 Oct 2023 11:01:14 -0700 Subject: [PATCH 10/89] [omicron-package] Retry failed downloads automatically (#4168) - Automatically retries downloads, with a small amount of constant backoff - Provides config options for users to customize these retry attempts / backoffs - Extends progress bars to show when download failures occur Tested manually, by running: ```bash cargo run --bin=omicron-package -- --retry-count 5 package ``` Then cycling my machine's network connectivity. I observed the "Failed to download prebuilt messages", saw the attempts ticking down, and then reconnected. Once connectivity was restored, the download succeeded. Fixes https://github.com/oxidecomputer/omicron/issues/4165 --- package/src/bin/omicron-package.rs | 163 ++++++++++++++++++++++------- 1 file changed, 125 insertions(+), 38 deletions(-) diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index a0146eee50..ea490e54cf 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -41,6 +41,11 @@ enum SubCommand { Deploy(DeployCommand), } +fn parse_duration_ms(arg: &str) -> Result { + let ms = arg.parse()?; + Ok(std::time::Duration::from_millis(ms)) +} + #[derive(Debug, Parser)] #[clap(name = "packaging tool")] struct Args { @@ -77,6 +82,23 @@ struct Args { )] force: bool, + #[clap( + long, + help = "Number of retries to use when re-attempting failed package downloads", + action, + default_value_t = 10 + )] + retry_count: usize, + + #[clap( + long, + help = "Duration, in ms, to wait before re-attempting failed package downloads", + action, + value_parser = parse_duration_ms, + default_value = "1000", + )] + retry_duration: std::time::Duration, + #[clap(subcommand)] subcommand: SubCommand, } @@ -303,8 +325,63 @@ async fn get_sha256_digest(path: &PathBuf) -> Result { Ok(context.finish()) } +async fn download_prebuilt( + progress: &PackageProgress, + package_name: &str, + repo: &str, + commit: &str, + expected_digest: &Vec, + path: &Path, +) -> Result<()> { + progress.set_message("downloading prebuilt".into()); + let url = format!( + "https://buildomat.eng.oxide.computer/public/file/oxidecomputer/{}/image/{}/{}", + repo, + commit, + path.file_name().unwrap().to_string_lossy(), + ); + let response = reqwest::Client::new() + .get(&url) + .send() + .await + .with_context(|| format!("failed to get {url}"))?; + progress.set_length( + response + .content_length() + .ok_or_else(|| anyhow!("Missing Content Length"))?, + ); + let mut file = tokio::fs::File::create(&path) + .await + .with_context(|| format!("failed to create {path:?}"))?; + let mut stream = response.bytes_stream(); + let mut context = DigestContext::new(&SHA256); + while let Some(chunk) = stream.next().await { + let chunk = chunk + .with_context(|| format!("failed reading response from {url}"))?; + // Update the running SHA digest + context.update(&chunk); + // Update the downloaded file + file.write_all(&chunk) + .await + .with_context(|| format!("failed writing {path:?}"))?; + // Record progress in the UI + progress.increment(chunk.len().try_into().unwrap()); + } + + let digest = context.finish(); + if digest.as_ref() != expected_digest { + bail!( + "Digest mismatch downloading {package_name}: Saw {}, expected {}", + hex::encode(digest.as_ref()), + hex::encode(expected_digest) + ); + } + Ok(()) +} + // Ensures a package exists, either by creating it or downloading it. async fn get_package( + config: &Config, target: &Target, ui: &Arc, package_name: &String, @@ -328,45 +405,30 @@ async fn get_package( }; if should_download { - progress.set_message("downloading prebuilt".into()); - let url = format!( - "https://buildomat.eng.oxide.computer/public/file/oxidecomputer/{}/image/{}/{}", - repo, - commit, - path.as_path().file_name().unwrap().to_string_lossy(), - ); - let response = reqwest::Client::new() - .get(&url) - .send() - .await - .with_context(|| format!("failed to get {url}"))?; - progress.set_length( - response - .content_length() - .ok_or_else(|| anyhow!("Missing Content Length"))?, - ); - let mut file = tokio::fs::File::create(&path) + let mut attempts_left = config.retry_count + 1; + loop { + match download_prebuilt( + &progress, + package_name, + repo, + commit, + &expected_digest, + path.as_path(), + ) .await - .with_context(|| format!("failed to create {path:?}"))?; - let mut stream = response.bytes_stream(); - let mut context = DigestContext::new(&SHA256); - while let Some(chunk) = stream.next().await { - let chunk = chunk.with_context(|| { - format!("failed reading response from {url}") - })?; - // Update the running SHA digest - context.update(&chunk); - // Update the downloaded file - file.write_all(&chunk) - .await - .with_context(|| format!("failed writing {path:?}"))?; - // Record progress in the UI - progress.increment(chunk.len().try_into().unwrap()); - } - - let digest = context.finish(); - if digest.as_ref() != expected_digest { - bail!("Digest mismatch downloading {package_name}: Saw {}, expected {}", hex::encode(digest.as_ref()), hex::encode(expected_digest)); + { + Ok(()) => break, + Err(err) => { + attempts_left -= 1; + let msg = format!("Failed to download prebuilt ({attempts_left} attempts remaining)"); + progress.set_error_message(msg.into()); + if attempts_left == 0 { + bail!("Failed to download package: {err}"); + } + tokio::time::sleep(config.retry_duration).await; + progress.reset(); + } + } } } } @@ -463,6 +525,7 @@ async fn do_package(config: &Config, output_directory: &Path) -> Result<()> { None, |((package_name, package), ui)| async move { get_package( + &config, &config.target, &ui, package_name, @@ -761,6 +824,13 @@ fn completed_progress_style() -> ProgressStyle { .progress_chars("#>.") } +fn error_progress_style() -> ProgressStyle { + ProgressStyle::default_bar() + .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg:.red}") + .expect("Invalid template") + .progress_chars("#>.") +} + // Struct managing display of progress to UI. struct ProgressUI { multi: MultiProgress, @@ -782,10 +852,21 @@ impl PackageProgress { fn set_length(&self, total: u64) { self.pb.set_length(total); } + + fn set_error_message(&self, message: std::borrow::Cow<'static, str>) { + self.pb.set_style(error_progress_style()); + self.pb.set_message(format!("{}: {}", self.service_name, message)); + self.pb.tick(); + } + + fn reset(&self) { + self.pb.reset(); + } } impl Progress for PackageProgress { fn set_message(&self, message: std::borrow::Cow<'static, str>) { + self.pb.set_style(in_progress_style()); self.pb.set_message(format!("{}: {}", self.service_name, message)); self.pb.tick(); } @@ -820,6 +901,10 @@ struct Config { target: Target, // True if we should skip confirmations for destructive operations. force: bool, + // Number of times to retry failed downloads. + retry_count: usize, + // Duration to wait before retrying failed downloads. + retry_duration: std::time::Duration, } impl Config { @@ -886,6 +971,8 @@ async fn main() -> Result<()> { package_config, target, force: args.force, + retry_count: args.retry_count, + retry_duration: args.retry_duration, }) }; From e9210fc2d9987fb6a880356a10a8308bc0f81678 Mon Sep 17 00:00:00 2001 From: Rain Date: Mon, 2 Oct 2023 11:13:16 -0700 Subject: [PATCH 11/89] [workspace-hack] record exact versions (#4171) Exact versions work better with dependabot. This can be reverted with renovate if we so choose. --- .config/hakari.toml | 5 +- workspace-hack/Cargo.toml | 446 +++++++++++++++++++------------------- 2 files changed, 226 insertions(+), 225 deletions(-) diff --git a/.config/hakari.toml b/.config/hakari.toml index 9562f92300..62f15df276 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -22,8 +22,9 @@ platforms = [ # "x86_64-pc-windows-msvc", ] +# Write out exact versions rather than a semver range. (Defaults to false.) +exact-versions = true + [traversal-excludes] workspace-members = ["xtask"] -# Write out exact versions rather than a semver range. (Defaults to false.) -# exact-versions = true diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index d3e00b1831..820b2d2336 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -14,248 +14,248 @@ publish = false ### BEGIN HAKARI SECTION [dependencies] -anyhow = { version = "1", features = ["backtrace"] } -bit-set = { version = "0.5" } -bit-vec = { version = "0.6" } -bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1" } -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["serde"] } -bitvec = { version = "1" } -bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2" } -bstr-dff4ba8e3ae991db = { package = "bstr", version = "1" } -bytes = { version = "1", features = ["serde"] } -chrono = { version = "0.4", features = ["alloc", "serde"] } -cipher = { version = "0.4", default-features = false, features = ["block-padding", "zeroize"] } -clap = { version = "4", features = ["derive", "env", "wrap_help"] } -clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "suggestions", "usage", "wrap_help"] } -console = { version = "0.15" } -const-oid = { version = "0.9", default-features = false, features = ["db", "std"] } -crossbeam-epoch = { version = "0.9" } -crossbeam-utils = { version = "0.8" } -crypto-common = { version = "0.1", default-features = false, features = ["getrandom", "std"] } -diesel = { version = "2", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } -digest = { version = "0.10", features = ["mac", "oid", "std"] } -either = { version = "1" } -flate2 = { version = "1" } -futures = { version = "0.3" } -futures-channel = { version = "0.3", features = ["sink"] } -futures-core = { version = "0.3" } -futures-io = { version = "0.3", default-features = false, features = ["std"] } -futures-sink = { version = "0.3" } -futures-task = { version = "0.3", default-features = false, features = ["std"] } -futures-util = { version = "0.3", features = ["channel", "io", "sink"] } +anyhow = { version = "1.0.75", features = ["backtrace"] } +bit-set = { version = "0.5.3" } +bit-vec = { version = "0.6.3" } +bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["serde"] } +bitvec = { version = "1.0.1" } +bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" } +bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" } +bytes = { version = "1.5.0", features = ["serde"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } +cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] } +clap = { version = "4.4.3", features = ["derive", "env", "wrap_help"] } +clap_builder = { version = "4.4.2", default-features = false, features = ["color", "env", "std", "suggestions", "usage", "wrap_help"] } +console = { version = "0.15.7" } +const-oid = { version = "0.9.5", default-features = false, features = ["db", "std"] } +crossbeam-epoch = { version = "0.9.15" } +crossbeam-utils = { version = "0.8.16" } +crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } +diesel = { version = "2.1.1", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } +digest = { version = "0.10.7", features = ["mac", "oid", "std"] } +either = { version = "1.9.0" } +flate2 = { version = "1.0.27" } +futures = { version = "0.3.28" } +futures-channel = { version = "0.3.28", features = ["sink"] } +futures-core = { version = "0.3.28" } +futures-io = { version = "0.3.28", default-features = false, features = ["std"] } +futures-sink = { version = "0.3.28" } +futures-task = { version = "0.3.28", default-features = false, features = ["std"] } +futures-util = { version = "0.3.28", features = ["channel", "io", "sink"] } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", features = ["std"] } -generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] } -getrandom = { version = "0.2", default-features = false, features = ["js", "rdrand", "std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } -hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13" } -hex = { version = "0.4", features = ["serde"] } -hyper = { version = "0.14", features = ["full"] } -indexmap = { version = "2", features = ["serde"] } -inout = { version = "0.1", default-features = false, features = ["std"] } -ipnetwork = { version = "0.20", features = ["schemars"] } -itertools = { version = "0.10" } -lalrpop-util = { version = "0.19" } -lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2", features = ["extra_traits"] } -log = { version = "0.4", default-features = false, features = ["std"] } -managed = { version = "0.8", default-features = false, features = ["alloc", "map"] } -memchr = { version = "2" } -num-bigint = { version = "0.4", features = ["rand"] } -num-integer = { version = "0.1", features = ["i128"] } -num-iter = { version = "0.1", default-features = false, features = ["i128"] } -num-traits = { version = "0.2", features = ["i128", "libm"] } -openapiv3 = { version = "1", default-features = false, features = ["skip_serializing_defaults"] } -petgraph = { version = "0.6", features = ["serde-1"] } -postgres-types = { version = "0.2", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } -ppv-lite86 = { version = "0.2", default-features = false, features = ["simd", "std"] } -predicates = { version = "3" } -rand = { version = "0.8", features = ["min_const_gen"] } -rand_chacha = { version = "0.3" } -regex = { version = "1" } -regex-automata = { version = "0.3", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } -regex-syntax = { version = "0.7" } -reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls", "stream"] } -ring = { version = "0.16", features = ["std"] } -schemars = { version = "0.8", features = ["bytes", "chrono", "uuid1"] } -semver = { version = "1", features = ["serde"] } -serde = { version = "1", features = ["alloc", "derive", "rc"] } -sha2 = { version = "0.10", features = ["oid"] } -signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] } -similar = { version = "2", features = ["inline", "unicode"] } -slog = { version = "2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } -spin = { version = "0.9" } -string_cache = { version = "0.8" } -subtle = { version = "2" } -syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "fold", "full", "visit"] } -syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } -textwrap = { version = "0.16" } -time = { version = "0.3", features = ["formatting", "local-offset", "macros", "parsing"] } -tokio = { version = "1", features = ["full", "test-util"] } -tokio-postgres = { version = "0.7", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } -tokio-stream = { version = "0.1", features = ["net"] } -toml = { version = "0.7" } -toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } -toml_edit = { version = "0.19", features = ["serde"] } -tracing = { version = "0.1", features = ["log"] } -trust-dns-proto = { version = "0.22" } -unicode-bidi = { version = "0.3" } -unicode-normalization = { version = "0.1" } -usdt = { version = "0.3" } -uuid = { version = "1", features = ["serde", "v4"] } -yasna = { version = "0.5", features = ["bit-vec", "num-bigint", "std", "time"] } -zeroize = { version = "1", features = ["std", "zeroize_derive"] } -zip = { version = "0.6", default-features = false, features = ["bzip2", "deflate"] } +generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } +getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } +hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } +hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } +hex = { version = "0.4.3", features = ["serde"] } +hyper = { version = "0.14.27", features = ["full"] } +indexmap = { version = "2.0.0", features = ["serde"] } +inout = { version = "0.1.3", default-features = false, features = ["std"] } +ipnetwork = { version = "0.20.0", features = ["schemars"] } +itertools = { version = "0.10.5" } +lalrpop-util = { version = "0.19.12" } +lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } +libc = { version = "0.2.148", features = ["extra_traits"] } +log = { version = "0.4.20", default-features = false, features = ["std"] } +managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] } +memchr = { version = "2.6.3" } +num-bigint = { version = "0.4.4", features = ["rand"] } +num-integer = { version = "0.1.45", features = ["i128"] } +num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } +num-traits = { version = "0.2.16", features = ["i128", "libm"] } +openapiv3 = { version = "1.0.3", default-features = false, features = ["skip_serializing_defaults"] } +petgraph = { version = "0.6.4", features = ["serde-1"] } +postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } +ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } +predicates = { version = "3.0.3" } +rand = { version = "0.8.5", features = ["min_const_gen"] } +rand_chacha = { version = "0.3.1" } +regex = { version = "1.9.5" } +regex-automata = { version = "0.3.8", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } +regex-syntax = { version = "0.7.5" } +reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } +ring = { version = "0.16.20", features = ["std"] } +schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } +semver = { version = "1.0.18", features = ["serde"] } +serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } +sha2 = { version = "0.10.7", features = ["oid"] } +signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } +similar = { version = "2.2.1", features = ["inline", "unicode"] } +slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } +spin = { version = "0.9.8" } +string_cache = { version = "0.8.7" } +subtle = { version = "2.5.0" } +syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] } +syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +textwrap = { version = "0.16.0" } +time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } +tokio = { version = "1.32.0", features = ["full", "test-util"] } +tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } +tokio-stream = { version = "0.1.14", features = ["net"] } +toml = { version = "0.7.8" } +toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } +toml_edit = { version = "0.19.15", features = ["serde"] } +tracing = { version = "0.1.37", features = ["log"] } +trust-dns-proto = { version = "0.22.0" } +unicode-bidi = { version = "0.3.13" } +unicode-normalization = { version = "0.1.22" } +usdt = { version = "0.3.5" } +uuid = { version = "1.4.1", features = ["serde", "v4"] } +yasna = { version = "0.5.2", features = ["bit-vec", "num-bigint", "std", "time"] } +zeroize = { version = "1.6.0", features = ["std", "zeroize_derive"] } +zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [build-dependencies] -anyhow = { version = "1", features = ["backtrace"] } -bit-set = { version = "0.5" } -bit-vec = { version = "0.6" } -bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1" } -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["serde"] } -bitvec = { version = "1" } -bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2" } -bstr-dff4ba8e3ae991db = { package = "bstr", version = "1" } -bytes = { version = "1", features = ["serde"] } -cc = { version = "1", default-features = false, features = ["parallel"] } -chrono = { version = "0.4", features = ["alloc", "serde"] } -cipher = { version = "0.4", default-features = false, features = ["block-padding", "zeroize"] } -clap = { version = "4", features = ["derive", "env", "wrap_help"] } -clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "suggestions", "usage", "wrap_help"] } -console = { version = "0.15" } -const-oid = { version = "0.9", default-features = false, features = ["db", "std"] } -crossbeam-epoch = { version = "0.9" } -crossbeam-utils = { version = "0.8" } -crypto-common = { version = "0.1", default-features = false, features = ["getrandom", "std"] } -diesel = { version = "2", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } -digest = { version = "0.10", features = ["mac", "oid", "std"] } -either = { version = "1" } -flate2 = { version = "1" } -futures = { version = "0.3" } -futures-channel = { version = "0.3", features = ["sink"] } -futures-core = { version = "0.3" } -futures-io = { version = "0.3", default-features = false, features = ["std"] } -futures-sink = { version = "0.3" } -futures-task = { version = "0.3", default-features = false, features = ["std"] } -futures-util = { version = "0.3", features = ["channel", "io", "sink"] } +anyhow = { version = "1.0.75", features = ["backtrace"] } +bit-set = { version = "0.5.3" } +bit-vec = { version = "0.6.3" } +bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["serde"] } +bitvec = { version = "1.0.1" } +bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" } +bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" } +bytes = { version = "1.5.0", features = ["serde"] } +cc = { version = "1.0.83", default-features = false, features = ["parallel"] } +chrono = { version = "0.4.31", features = ["alloc", "serde"] } +cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] } +clap = { version = "4.4.3", features = ["derive", "env", "wrap_help"] } +clap_builder = { version = "4.4.2", default-features = false, features = ["color", "env", "std", "suggestions", "usage", "wrap_help"] } +console = { version = "0.15.7" } +const-oid = { version = "0.9.5", default-features = false, features = ["db", "std"] } +crossbeam-epoch = { version = "0.9.15" } +crossbeam-utils = { version = "0.8.16" } +crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } +diesel = { version = "2.1.1", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } +digest = { version = "0.10.7", features = ["mac", "oid", "std"] } +either = { version = "1.9.0" } +flate2 = { version = "1.0.27" } +futures = { version = "0.3.28" } +futures-channel = { version = "0.3.28", features = ["sink"] } +futures-core = { version = "0.3.28" } +futures-io = { version = "0.3.28", default-features = false, features = ["std"] } +futures-sink = { version = "0.3.28" } +futures-task = { version = "0.3.28", default-features = false, features = ["std"] } +futures-util = { version = "0.3.28", features = ["channel", "io", "sink"] } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", features = ["std"] } -generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] } -getrandom = { version = "0.2", default-features = false, features = ["js", "rdrand", "std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } -hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13" } -hex = { version = "0.4", features = ["serde"] } -hyper = { version = "0.14", features = ["full"] } -indexmap = { version = "2", features = ["serde"] } -inout = { version = "0.1", default-features = false, features = ["std"] } -ipnetwork = { version = "0.20", features = ["schemars"] } -itertools = { version = "0.10" } -lalrpop-util = { version = "0.19" } -lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2", features = ["extra_traits"] } -log = { version = "0.4", default-features = false, features = ["std"] } -managed = { version = "0.8", default-features = false, features = ["alloc", "map"] } -memchr = { version = "2" } -num-bigint = { version = "0.4", features = ["rand"] } -num-integer = { version = "0.1", features = ["i128"] } -num-iter = { version = "0.1", default-features = false, features = ["i128"] } -num-traits = { version = "0.2", features = ["i128", "libm"] } -openapiv3 = { version = "1", default-features = false, features = ["skip_serializing_defaults"] } -petgraph = { version = "0.6", features = ["serde-1"] } -postgres-types = { version = "0.2", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } -ppv-lite86 = { version = "0.2", default-features = false, features = ["simd", "std"] } -predicates = { version = "3" } -rand = { version = "0.8", features = ["min_const_gen"] } -rand_chacha = { version = "0.3" } -regex = { version = "1" } -regex-automata = { version = "0.3", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } -regex-syntax = { version = "0.7" } -reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls", "stream"] } -ring = { version = "0.16", features = ["std"] } -schemars = { version = "0.8", features = ["bytes", "chrono", "uuid1"] } -semver = { version = "1", features = ["serde"] } -serde = { version = "1", features = ["alloc", "derive", "rc"] } -sha2 = { version = "0.10", features = ["oid"] } -signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] } -similar = { version = "2", features = ["inline", "unicode"] } -slog = { version = "2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } -spin = { version = "0.9" } -string_cache = { version = "0.8" } -subtle = { version = "2" } -syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "fold", "full", "visit"] } -syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } -textwrap = { version = "0.16" } -time = { version = "0.3", features = ["formatting", "local-offset", "macros", "parsing"] } -time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing"] } -tokio = { version = "1", features = ["full", "test-util"] } -tokio-postgres = { version = "0.7", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } -tokio-stream = { version = "0.1", features = ["net"] } -toml = { version = "0.7" } -toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } -toml_edit = { version = "0.19", features = ["serde"] } -tracing = { version = "0.1", features = ["log"] } -trust-dns-proto = { version = "0.22" } -unicode-bidi = { version = "0.3" } -unicode-normalization = { version = "0.1" } -unicode-xid = { version = "0.2" } -usdt = { version = "0.3" } -uuid = { version = "1", features = ["serde", "v4"] } -yasna = { version = "0.5", features = ["bit-vec", "num-bigint", "std", "time"] } -zeroize = { version = "1", features = ["std", "zeroize_derive"] } -zip = { version = "0.6", default-features = false, features = ["bzip2", "deflate"] } +generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } +getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } +hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } +hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } +hex = { version = "0.4.3", features = ["serde"] } +hyper = { version = "0.14.27", features = ["full"] } +indexmap = { version = "2.0.0", features = ["serde"] } +inout = { version = "0.1.3", default-features = false, features = ["std"] } +ipnetwork = { version = "0.20.0", features = ["schemars"] } +itertools = { version = "0.10.5" } +lalrpop-util = { version = "0.19.12" } +lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } +libc = { version = "0.2.148", features = ["extra_traits"] } +log = { version = "0.4.20", default-features = false, features = ["std"] } +managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] } +memchr = { version = "2.6.3" } +num-bigint = { version = "0.4.4", features = ["rand"] } +num-integer = { version = "0.1.45", features = ["i128"] } +num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } +num-traits = { version = "0.2.16", features = ["i128", "libm"] } +openapiv3 = { version = "1.0.3", default-features = false, features = ["skip_serializing_defaults"] } +petgraph = { version = "0.6.4", features = ["serde-1"] } +postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } +ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } +predicates = { version = "3.0.3" } +rand = { version = "0.8.5", features = ["min_const_gen"] } +rand_chacha = { version = "0.3.1" } +regex = { version = "1.9.5" } +regex-automata = { version = "0.3.8", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } +regex-syntax = { version = "0.7.5" } +reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } +ring = { version = "0.16.20", features = ["std"] } +schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } +semver = { version = "1.0.18", features = ["serde"] } +serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } +sha2 = { version = "0.10.7", features = ["oid"] } +signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } +similar = { version = "2.2.1", features = ["inline", "unicode"] } +slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } +spin = { version = "0.9.8" } +string_cache = { version = "0.8.7" } +subtle = { version = "2.5.0" } +syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] } +syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +textwrap = { version = "0.16.0" } +time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } +time-macros = { version = "0.2.13", default-features = false, features = ["formatting", "parsing"] } +tokio = { version = "1.32.0", features = ["full", "test-util"] } +tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } +tokio-stream = { version = "0.1.14", features = ["net"] } +toml = { version = "0.7.8" } +toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } +toml_edit = { version = "0.19.15", features = ["serde"] } +tracing = { version = "0.1.37", features = ["log"] } +trust-dns-proto = { version = "0.22.0" } +unicode-bidi = { version = "0.3.13" } +unicode-normalization = { version = "0.1.22" } +unicode-xid = { version = "0.2.4" } +usdt = { version = "0.3.5" } +uuid = { version = "1.4.1", features = ["serde", "v4"] } +yasna = { version = "0.5.2", features = ["bit-vec", "num-bigint", "std", "time"] } +zeroize = { version = "1.6.0", features = ["std", "zeroize_derive"] } +zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [target.x86_64-unknown-linux-gnu.dependencies] -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24" } -mio = { version = "0.8", features = ["net", "os-ext"] } -once_cell = { version = "1", features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24.1" } +mio = { version = "0.8.8", features = ["net", "os-ext"] } +once_cell = { version = "1.18.0", features = ["unstable"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-unknown-linux-gnu.build-dependencies] -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24" } -mio = { version = "0.8", features = ["net", "os-ext"] } -once_cell = { version = "1", features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24.1" } +mio = { version = "0.8.8", features = ["net", "os-ext"] } +once_cell = { version = "1.18.0", features = ["unstable"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-apple-darwin.dependencies] -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24" } -mio = { version = "0.8", features = ["net", "os-ext"] } -once_cell = { version = "1", features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24.1" } +mio = { version = "0.8.8", features = ["net", "os-ext"] } +once_cell = { version = "1.18.0", features = ["unstable"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-apple-darwin.build-dependencies] -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24" } -mio = { version = "0.8", features = ["net", "os-ext"] } -once_cell = { version = "1", features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24.1" } +mio = { version = "0.8.8", features = ["net", "os-ext"] } +once_cell = { version = "1.18.0", features = ["unstable"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.aarch64-apple-darwin.dependencies] -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24" } -mio = { version = "0.8", features = ["net", "os-ext"] } -once_cell = { version = "1", features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24.1" } +mio = { version = "0.8.8", features = ["net", "os-ext"] } +once_cell = { version = "1.18.0", features = ["unstable"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.aarch64-apple-darwin.build-dependencies] -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24" } -mio = { version = "0.8", features = ["net", "os-ext"] } -once_cell = { version = "1", features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24.1" } +mio = { version = "0.8.8", features = ["net", "os-ext"] } +once_cell = { version = "1.18.0", features = ["unstable"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-unknown-illumos.dependencies] -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24" } -mio = { version = "0.8", features = ["net", "os-ext"] } -once_cell = { version = "1", features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24.1" } +mio = { version = "0.8.8", features = ["net", "os-ext"] } +once_cell = { version = "1.18.0", features = ["unstable"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-unknown-illumos.build-dependencies] -bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24" } -mio = { version = "0.8", features = ["net", "os-ext"] } -once_cell = { version = "1", features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } +bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } +hyper-rustls = { version = "0.24.1" } +mio = { version = "0.8.8", features = ["net", "os-ext"] } +once_cell = { version = "1.18.0", features = ["unstable"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } ### END HAKARI SECTION From e86579ce0f16021d2bbb84be4e97a8d24b61acb3 Mon Sep 17 00:00:00 2001 From: bnaecker Date: Mon, 2 Oct 2023 14:06:05 -0700 Subject: [PATCH 12/89] Fix path checks for archived log files (#4161) - Fixes #4160. - Checks for archived log files were wrong. This used the SMF FMRI, rather than the derived log filename, which translates slashes into dashes. This separates checks for Oxide-managed FMRIs and the log files for those. - Use the _log file_ check when looking for archived files. - Add tests for both checks and the method for finding archived log files for a service. --- Cargo.lock | 1 + illumos-utils/src/running_zone.rs | 55 +++++++++++++++++++++---- sled-agent/Cargo.toml | 1 + sled-agent/src/zone_bundle.rs | 67 +++++++++++++++++++++++++++++-- 4 files changed, 113 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7296ea184..3a45dcb381 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5299,6 +5299,7 @@ dependencies = [ "static_assertions", "subprocess", "tar", + "tempfile", "thiserror", "tofino", "tokio", diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index 4d3481b6c3..734f22bd30 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -987,7 +987,7 @@ impl RunningZone { let output = self.run_cmd(&["svcs", "-H", "-o", "fmri"])?; Ok(output .lines() - .filter(|line| is_oxide_smf_log_file(line)) + .filter(|line| is_oxide_smf_service(line)) .map(|line| line.trim().to_string()) .collect()) } @@ -1267,10 +1267,51 @@ impl InstalledZone { } } -/// Return true if the named file appears to be a log file for an Oxide SMF -/// service. -pub fn is_oxide_smf_log_file(name: impl AsRef) -> bool { - const SMF_SERVICE_PREFIXES: [&str; 2] = ["/oxide", "/system/illumos"]; - let name = name.as_ref(); - SMF_SERVICE_PREFIXES.iter().any(|needle| name.contains(needle)) +/// Return true if the service with the given FMRI appears to be an +/// Oxide-managed service. +pub fn is_oxide_smf_service(fmri: impl AsRef) -> bool { + const SMF_SERVICE_PREFIXES: [&str; 2] = + ["svc:/oxide/", "svc:/system/illumos/"]; + let fmri = fmri.as_ref(); + SMF_SERVICE_PREFIXES.iter().any(|prefix| fmri.starts_with(prefix)) +} + +/// Return true if the provided file name appears to be a valid log file for an +/// Oxide-managed SMF service. +/// +/// Note that this operates on the _file name_. Any leading path components will +/// cause this check to return `false`. +pub fn is_oxide_smf_log_file(filename: impl AsRef) -> bool { + // Log files are named by the SMF services, with the `/` in the FMRI + // translated to a `-`. + const PREFIXES: [&str; 2] = ["oxide-", "system-illumos-"]; + let filename = filename.as_ref(); + PREFIXES + .iter() + .any(|prefix| filename.starts_with(prefix) && filename.contains(".log")) +} + +#[cfg(test)] +mod tests { + use super::is_oxide_smf_log_file; + use super::is_oxide_smf_service; + + #[test] + fn test_is_oxide_smf_service() { + assert!(is_oxide_smf_service("svc:/oxide/blah:default")); + assert!(is_oxide_smf_service("svc:/system/illumos/blah:default")); + assert!(!is_oxide_smf_service("svc:/system/blah:default")); + assert!(!is_oxide_smf_service("svc:/not/oxide/blah:default")); + } + + #[test] + fn test_is_oxide_smf_log_file() { + assert!(is_oxide_smf_log_file("oxide-blah:default.log")); + assert!(is_oxide_smf_log_file("oxide-blah:default.log.0")); + assert!(is_oxide_smf_log_file("oxide-blah:default.log.1111")); + assert!(is_oxide_smf_log_file("system-illumos-blah:default.log")); + assert!(is_oxide_smf_log_file("system-illumos-blah:default.log.0")); + assert!(!is_oxide_smf_log_file("not-oxide-blah:default.log")); + assert!(!is_oxide_smf_log_file("not-system-illumos-blah:default.log")); + } } diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index b131698395..88e51a3bc3 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -95,6 +95,7 @@ serial_test.workspace = true subprocess.workspace = true slog-async.workspace = true slog-term.workspace = true +tempfile.workspace = true illumos-utils = { workspace = true, features = ["testing"] } diff --git a/sled-agent/src/zone_bundle.rs b/sled-agent/src/zone_bundle.rs index 2eeb8ebe7d..4c2d6a4113 100644 --- a/sled-agent/src/zone_bundle.rs +++ b/sled-agent/src/zone_bundle.rs @@ -899,9 +899,9 @@ async fn find_archived_log_files( continue; }; let fname = path.file_name().unwrap(); - if is_oxide_smf_log_file(fname) - && fname.contains(svc_name) - { + let is_oxide = is_oxide_smf_log_file(fname); + let contains = fname.contains(svc_name); + if is_oxide && contains { debug!( log, "found archived log file"; @@ -910,6 +910,14 @@ async fn find_archived_log_files( "path" => ?path, ); files.push(path); + } else { + debug!( + log, + "skipping non-matching log file"; + "filename" => fname, + "is_oxide_smf_log_file" => is_oxide, + "contains_svc_name" => contains, + ); } } Err(e) => { @@ -1764,6 +1772,7 @@ mod tests { #[cfg(all(target_os = "illumos", test))] mod illumos_tests { + use super::find_archived_log_files; use super::zfs_quota; use super::CleanupContext; use super::CleanupPeriod; @@ -1852,12 +1861,17 @@ mod illumos_tests { } } - async fn setup_fake_cleanup_task() -> anyhow::Result { + fn test_logger() -> Logger { let dec = slog_term::PlainSyncDecorator::new(slog_term::TestStdoutWriter); let drain = slog_term::FullFormat::new(dec).build().fuse(); let log = Logger::root(drain, slog::o!("component" => "fake-cleanup-task")); + log + } + + async fn setup_fake_cleanup_task() -> anyhow::Result { + let log = test_logger(); let context = CleanupContext::default(); let resource_wrapper = ResourceWrapper::new().await; let bundler = @@ -2279,4 +2293,49 @@ mod illumos_tests { let bytes = tokio::fs::metadata(&path).await?.len(); Ok(ZoneBundleInfo { metadata, path, bytes }) } + + #[tokio::test] + async fn test_find_archived_log_files() { + let log = test_logger(); + let tmpdir = tempfile::tempdir().expect("Failed to make tempdir"); + + let mut should_match = [ + "oxide-foo:default.log", + "oxide-foo:default.log.1000", + "system-illumos-foo:default.log", + "system-illumos-foo:default.log.100", + ]; + let should_not_match = [ + "oxide-foo:default", + "not-oxide-foo:default.log.1000", + "system-illumos-foo", + "not-system-illumos-foo:default.log.100", + ]; + for name in should_match.iter().chain(should_not_match.iter()) { + let path = tmpdir.path().join(name); + tokio::fs::File::create(path) + .await + .expect("failed to create dummy file"); + } + + let path = + Utf8PathBuf::try_from(tmpdir.path().as_os_str().to_str().unwrap()) + .unwrap(); + let mut files = find_archived_log_files( + &log, + "zone-name", // unused here, for logging only + "foo", + &[path], + ) + .await; + + // Sort everything to compare correctly. + should_match.sort(); + files.sort(); + assert_eq!(files.len(), should_match.len()); + assert!(files + .iter() + .zip(should_match.iter()) + .all(|(file, name)| { file.file_name().unwrap() == *name })); + } } From 6bc5e6062df7bffbe0eca1465b9d116ff7849e9f Mon Sep 17 00:00:00 2001 From: artemis everfree Date: Mon, 2 Oct 2023 19:29:36 -0700 Subject: [PATCH 13/89] RandomnWithDistinctSleds region allocation strategy (#3858) PR #3650 introduced the Random region allocation strategy to allocate regions randomly across the rack. This expands on that with the addition of the RandomWithDistinctSleds region allocation strategy. This strategy is the same, but requires the 3 crucible regions be allocated on 3 different sleds to improve resiliency against a whole-sled failure. The Random strategy still exists, and does not require 3 distinct sleds. This is useful in one-sled environments such as the integration tests, and lab setups. This also fixes a shortcoming of #3650 whereby multiple datasets on a single zpool could be selected. That fix applies to both the old Random strategy and the new RandomWithDistinctSleds strategy. In the present, I have unit tests that verify the allocation behavior works correctly with cockroachdb, and we can try it out on dogfood. Adds the `-r` / `--rack-topology` command line argument to omicron-package target create. Use this to specify whether you are packaging for a single-sled or multi-sled environment. Under single-sled environments, the requirement for 3 distinct sleds is removed. Fixes #3702 --------- Co-authored-by: iliana etaoin --- .github/buildomat/jobs/deploy.sh | 1 + .github/buildomat/jobs/package.sh | 9 +- .github/buildomat/jobs/tuf-repo.sh | 3 +- .github/workflows/rust.yml | 2 +- common/src/nexus_config.rs | 74 ++++- docs/how-to-run.adoc | 34 ++- installinator/Cargo.toml | 1 + .../db-model/src/queries/region_allocation.rs | 22 ++ nexus/db-queries/src/db/datastore/mod.rs | 277 +++++++++++++----- nexus/db-queries/src/db/datastore/region.rs | 2 +- .../src/db/queries/region_allocation.rs | 242 +++++++++------ nexus/examples/config.toml | 11 + nexus/src/app/mod.rs | 8 + nexus/src/app/sagas/disk_create.rs | 8 +- nexus/src/app/sagas/snapshot_create.rs | 7 +- nexus/tests/config.test.toml | 5 + package-manifest.toml | 3 +- package/src/bin/omicron-package.rs | 3 +- package/src/lib.rs | 23 ++ package/src/target.rs | 31 +- sled-agent/Cargo.toml | 2 + sled-agent/src/services.rs | 3 + smf/nexus/multi-sled/config-partial.toml | 45 +++ .../{ => single-sled}/config-partial.toml | 5 + 24 files changed, 617 insertions(+), 204 deletions(-) create mode 100644 smf/nexus/multi-sled/config-partial.toml rename smf/nexus/{ => single-sled}/config-partial.toml (86%) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 5d3dd8ec39..c2579d98ea 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -143,6 +143,7 @@ cd /opt/oxide/work ptime -m tar xvzf /input/package/work/package.tar.gz cp /input/package/work/zones/* out/ +mv out/omicron-nexus-single-sled.tar.gz out/omicron-nexus.tar.gz mkdir tests for p in /input/ci-tools/work/end-to-end-tests/*.gz; do ptime -m gunzip < "$p" > "tests/$(basename "${p%.gz}")" diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index fe5d6a9b7f..64c087524e 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -45,7 +45,7 @@ ptime -m ./tools/ci_download_softnpu_machinery # Build the test target ptime -m cargo run --locked --release --bin omicron-package -- \ - -t test target create -i standard -m non-gimlet -s softnpu + -t test target create -i standard -m non-gimlet -s softnpu -r single-sled ptime -m cargo run --locked --release --bin omicron-package -- \ -t test package @@ -81,9 +81,13 @@ stamp_packages() { done } +# Keep the single-sled Nexus zone around for the deploy job. (The global zone +# build below overwrites the file.) +mv out/omicron-nexus.tar.gz out/omicron-nexus-single-sled.tar.gz + # Build necessary for the global zone ptime -m cargo run --locked --release --bin omicron-package -- \ - -t host target create -i standard -m gimlet -s asic + -t host target create -i standard -m gimlet -s asic -r multi-sled ptime -m cargo run --locked --release --bin omicron-package -- \ -t host package stamp_packages omicron-sled-agent maghemite propolis-server overlay @@ -111,6 +115,7 @@ zones=( out/external-dns.tar.gz out/internal-dns.tar.gz out/omicron-nexus.tar.gz + out/omicron-nexus-single-sled.tar.gz out/oximeter-collector.tar.gz out/propolis-server.tar.gz out/switch-*.tar.gz diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index fab6770564..e169bebff6 100644 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -77,10 +77,11 @@ done mkdir /work/package pushd /work/package tar xf /input/package/work/package.tar.gz out package-manifest.toml target/release/omicron-package -target/release/omicron-package -t default target create -i standard -m gimlet -s asic +target/release/omicron-package -t default target create -i standard -m gimlet -s asic -r multi-sled ln -s /input/package/work/zones/* out/ rm out/switch-softnpu.tar.gz # not used when target switch=asic rm out/omicron-gateway-softnpu.tar.gz # not used when target switch=asic +rm out/omicron-nexus-single-sled.tar.gz # only used for deploy tests for zone in out/*.tar.gz; do target/release/omicron-package stamp "$(basename "${zone%.tar.gz}")" "$VERSION" done diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 722aacbe0f..f5cf1dc885 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -41,7 +41,7 @@ jobs: - name: Install Pre-Requisites run: ./tools/install_builder_prerequisites.sh -y - name: Set default target - run: cargo run --bin omicron-package -- -t default target create + run: cargo run --bin omicron-package -- -t default target create -r single-sled - name: Check build of deployed Omicron packages run: cargo run --bin omicron-package -- -t default check diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index 73ccec996c..ad62c34f92 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -372,6 +372,8 @@ pub struct PackageConfig { pub dendrite: HashMap, /// Background task configuration pub background_tasks: BackgroundTaskConfig, + /// Default Crucible region allocation strategy + pub default_region_allocation_strategy: RegionAllocationStrategy, } #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] @@ -594,6 +596,9 @@ mod test { dns_external.period_secs_propagation = 7 dns_external.max_concurrent_server_updates = 8 external_endpoints.period_secs = 9 + [default_region_allocation_strategy] + type = "random" + seed = 0 "##, ) .unwrap(); @@ -677,6 +682,10 @@ mod test { period_secs: Duration::from_secs(9), } }, + default_region_allocation_strategy: + crate::nexus_config::RegionAllocationStrategy::Random { + seed: Some(0) + } }, } ); @@ -724,6 +733,8 @@ mod test { dns_external.period_secs_propagation = 7 dns_external.max_concurrent_server_updates = 8 external_endpoints.period_secs = 9 + [default_region_allocation_strategy] + type = "random" "##, ) .unwrap(); @@ -864,25 +875,31 @@ mod test { struct DummyConfig { deployment: DeploymentConfig, } - let config_path = "../smf/nexus/config-partial.toml"; - println!( - "checking {:?} with example deployment section added", - config_path - ); - let mut contents = std::fs::read_to_string(config_path) - .expect("failed to read Nexus SMF config file"); - contents.push_str( - "\n\n\n \ - # !! content below added by test_repo_configs_are_valid()\n\ - \n\n\n", - ); let example_deployment = toml::to_string_pretty(&DummyConfig { deployment: example_config.deployment, }) .unwrap(); - contents.push_str(&example_deployment); - let _: Config = toml::from_str(&contents) - .expect("Nexus SMF config file is not valid"); + + let nexus_config_paths = [ + "../smf/nexus/single-sled/config-partial.toml", + "../smf/nexus/multi-sled/config-partial.toml", + ]; + for config_path in nexus_config_paths { + println!( + "checking {:?} with example deployment section added", + config_path + ); + let mut contents = std::fs::read_to_string(config_path) + .expect("failed to read Nexus SMF config file"); + contents.push_str( + "\n\n\n \ + # !! content below added by test_repo_configs_are_valid()\n\ + \n\n\n", + ); + contents.push_str(&example_deployment); + let _: Config = toml::from_str(&contents) + .expect("Nexus SMF config file is not valid"); + } } #[test] @@ -894,3 +911,30 @@ mod test { ); } } + +/// Defines a strategy for choosing what physical disks to use when allocating +/// new crucible regions. +/// +/// NOTE: More strategies can - and should! - be added. +/// +/// See for a more +/// complete discussion. +/// +/// Longer-term, we should consider: +/// - Storage size + remaining free space +/// - Sled placement of datasets +/// - What sort of loads we'd like to create (even split across all disks +/// may not be preferable, especially if maintenance is expected) +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum RegionAllocationStrategy { + /// Choose disks pseudo-randomly. An optional seed may be provided to make + /// the ordering deterministic, otherwise the current time in nanoseconds + /// will be used. Ordering is based on sorting the output of `md5(UUID of + /// candidate dataset + seed)`. The seed does not need to come from a + /// cryptographically secure source. + Random { seed: Option }, + + /// Like Random, but ensures that each region is allocated on its own sled. + RandomWithDistinctSleds { seed: Option }, +} diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 1988a42669..7539c5183f 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -321,10 +321,32 @@ Error: Creates a new build target, and sets it as "active" Usage: omicron-package target create [OPTIONS] Options: - -i, --image [default: standard] [possible values: standard, trampoline] - -m, --machine [possible values: gimlet, gimlet-standalone, non-gimlet] - -s, --switch [possible values: asic, stub, softnpu] - -h, --help Print help (see more with '--help') + -i, --image + [default: standard] + + Possible values: + - standard: A typical host OS image + - trampoline: A recovery host OS image, intended to bootstrap a Standard image + + -m, --machine + Possible values: + - gimlet: Use sled agent configuration for a Gimlet + - gimlet-standalone: Use sled agent configuration for a Gimlet running in isolation + - non-gimlet: Use sled agent configuration for a device emulating a Gimlet + + -s, --switch + Possible values: + - asic: Use the "real" Dendrite, that attempts to interact with the Tofino + - stub: Use a "stub" Dendrite that does not require any real hardware + - softnpu: Use a "softnpu" Dendrite that uses the SoftNPU asic emulator + + -r, --rack-topology + Possible values: + - multi-sled: Use configurations suitable for a multi-sled deployment, such as dogfood and production racks + - single-sled: Use configurations suitable for a single-sled deployment, such as CI and dev machines + + -h, --help + Print help (see a summary with '-h') ---- @@ -332,9 +354,9 @@ To set up a build target for a non-Gimlet machine with simulated (but fully func [source,console] ---- -$ cargo run --release --bin omicron-package -- -t default target create -i standard -m non-gimlet -s softnpu +$ cargo run --release --bin omicron-package -- -t default target create -i standard -m non-gimlet -s softnpu -r single-sled Finished release [optimized] target(s) in 0.66s - Running `target/release/omicron-package -t default target create -i standard -m non-gimlet -s softnpu` + Running `target/release/omicron-package -t default target create -i standard -m non-gimlet -s softnpu -r single-sled` Created new build target 'default' and set it as active ---- diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml index 3b2f04c38f..428ea0d08e 100644 --- a/installinator/Cargo.toml +++ b/installinator/Cargo.toml @@ -57,3 +57,4 @@ tokio-stream.workspace = true [features] image-standard = [] image-trampoline = [] +rack-topology-single-sled = [] \ No newline at end of file diff --git a/nexus/db-model/src/queries/region_allocation.rs b/nexus/db-model/src/queries/region_allocation.rs index 43fac3c9a6..2025e79fb8 100644 --- a/nexus/db-model/src/queries/region_allocation.rs +++ b/nexus/db-model/src/queries/region_allocation.rs @@ -47,6 +47,13 @@ table! { } } +table! { + shuffled_candidate_datasets { + id -> Uuid, + pool_id -> Uuid, + } +} + table! { candidate_regions { id -> Uuid, @@ -89,6 +96,19 @@ table! { } } +table! { + one_zpool_per_sled (pool_id) { + pool_id -> Uuid + } +} + +table! { + one_dataset_per_zpool { + id -> Uuid, + pool_id -> Uuid + } +} + table! { inserted_regions { id -> Uuid, @@ -141,6 +161,7 @@ diesel::allow_tables_to_appear_in_same_query!( ); diesel::allow_tables_to_appear_in_same_query!(old_regions, dataset,); +diesel::allow_tables_to_appear_in_same_query!(old_regions, zpool,); diesel::allow_tables_to_appear_in_same_query!( inserted_regions, @@ -149,6 +170,7 @@ diesel::allow_tables_to_appear_in_same_query!( diesel::allow_tables_to_appear_in_same_query!(candidate_zpools, dataset,); diesel::allow_tables_to_appear_in_same_query!(candidate_zpools, zpool,); +diesel::allow_tables_to_appear_in_same_query!(candidate_datasets, dataset); // == Needed for random region allocation == diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index ff1df710bb..b1f3203c60 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -307,43 +307,6 @@ pub enum UpdatePrecondition { Value(T), } -/// Defines a strategy for choosing what physical disks to use when allocating -/// new crucible regions. -/// -/// NOTE: More strategies can - and should! - be added. -/// -/// See for a more -/// complete discussion. -/// -/// Longer-term, we should consider: -/// - Storage size + remaining free space -/// - Sled placement of datasets -/// - What sort of loads we'd like to create (even split across all disks -/// may not be preferable, especially if maintenance is expected) -#[derive(Debug, Clone)] -pub enum RegionAllocationStrategy { - /// Choose disks that have the least data usage in the rack. This strategy - /// can lead to bad failure states wherein the disks with the least usage - /// have the least usage because regions on them are actually failing in - /// some way. Further retried allocations will then continue to try to - /// allocate onto the disk, perpetuating the problem. Currently this - /// strategy only exists so we can test that using different allocation - /// strategies actually results in different allocation patterns, hence the - /// `#[cfg(test)]`. - /// - /// See https://github.com/oxidecomputer/omicron/issues/3416 for more on the - /// failure-states associated with this strategy - #[cfg(test)] - LeastUsedDisk, - - /// Choose disks pseudo-randomly. An optional seed may be provided to make - /// the ordering deterministic, otherwise the current time in nanoseconds - /// will be used. Ordering is based on sorting the output of `md5(UUID of - /// candidate dataset + seed)`. The seed does not need to come from a - /// cryptographically secure source. - Random(Option), -} - /// Constructs a DataStore for use in test suites that has preloaded the /// built-in users, roles, and role assignments that are needed for basic /// operation @@ -421,7 +384,9 @@ mod test { use omicron_common::api::external::{ self, ByteCount, Error, IdentityMetadataCreateParams, LookupType, Name, }; + use omicron_common::nexus_config::RegionAllocationStrategy; use omicron_test_utils::dev; + use std::collections::HashMap; use std::collections::HashSet; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6}; use std::num::NonZeroU32; @@ -704,12 +669,18 @@ mod test { } } + struct TestDataset { + sled_id: Uuid, + dataset_id: Uuid, + } + async fn create_test_datasets_for_region_allocation( opctx: &OpContext, datastore: Arc, - ) -> Vec { + number_of_sleds: usize, + ) -> Vec { // Create sleds... - let sled_ids: Vec = stream::iter(0..REGION_REDUNDANCY_THRESHOLD) + let sled_ids: Vec = stream::iter(0..number_of_sleds) .then(|_| create_test_sled(&datastore)) .collect() .await; @@ -740,48 +711,69 @@ mod test { .collect() .await; + #[derive(Copy, Clone)] + struct Zpool { + sled_id: Uuid, + pool_id: Uuid, + } + // 1 pool per disk - let zpool_ids: Vec = stream::iter(physical_disks) + let zpools: Vec = stream::iter(physical_disks) .then(|disk| { - create_test_zpool(&datastore, disk.sled_id, disk.disk_id) + let pool_id_future = + create_test_zpool(&datastore, disk.sled_id, disk.disk_id); + async move { + let pool_id = pool_id_future.await; + Zpool { sled_id: disk.sled_id, pool_id } + } }) .collect() .await; let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0); - // 1 dataset per zpool - let dataset_ids: Vec = stream::iter(zpool_ids) - .then(|zpool_id| { - let id = Uuid::new_v4(); - let dataset = Dataset::new( - id, - zpool_id, - bogus_addr, - DatasetKind::Crucible, - ); - let datastore = datastore.clone(); - async move { - datastore.dataset_upsert(dataset).await.unwrap(); - id - } + let datasets: Vec = stream::iter(zpools) + .map(|zpool| { + // 3 datasets per zpool, to test that pools are distinct + let zpool_iter: Vec = (0..3).map(|_| zpool).collect(); + stream::iter(zpool_iter).then(|zpool| { + let id = Uuid::new_v4(); + let dataset = Dataset::new( + id, + zpool.pool_id, + bogus_addr, + DatasetKind::Crucible, + ); + + let datastore = datastore.clone(); + async move { + datastore.dataset_upsert(dataset).await.unwrap(); + + TestDataset { sled_id: zpool.sled_id, dataset_id: id } + } + }) }) + .flatten() .collect() .await; - dataset_ids + datasets } #[tokio::test] /// Note that this test is currently non-deterministic. It can be made /// deterministic by generating deterministic *dataset* Uuids. The sled and /// pool IDs should not matter. - async fn test_region_allocation() { - let logctx = dev::test_setup_log("test_region_allocation"); + async fn test_region_allocation_strat_random() { + let logctx = dev::test_setup_log("test_region_allocation_strat_random"); let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; - create_test_datasets_for_region_allocation(&opctx, datastore.clone()) - .await; + create_test_datasets_for_region_allocation( + &opctx, + datastore.clone(), + REGION_REDUNDANCY_THRESHOLD, + ) + .await; // Allocate regions from the datasets for this disk. Do it a few times // for good measure. @@ -799,7 +791,9 @@ mod test { volume_id, ¶ms.disk_source, params.size, - &RegionAllocationStrategy::Random(Some(alloc_seed as u128)), + &RegionAllocationStrategy::Random { + seed: Some(alloc_seed), + }, ) .await .unwrap(); @@ -809,8 +803,81 @@ mod test { let mut disk_datasets = HashSet::new(); let mut disk_zpools = HashSet::new(); - // TODO: When allocation chooses 3 distinct sleds, uncomment this. - // let mut disk1_sleds = HashSet::new(); + for (dataset, region) in dataset_and_regions { + // Must be 3 unique datasets + assert!(disk_datasets.insert(dataset.id())); + + // Must be 3 unique zpools + assert!(disk_zpools.insert(dataset.pool_id)); + + assert_eq!(volume_id, region.volume_id()); + assert_eq!(ByteCount::from(4096), region.block_size()); + let (_, extent_count) = DataStore::get_crucible_allocation( + &BlockSize::AdvancedFormat, + params.size, + ); + assert_eq!(extent_count, region.extent_count()); + } + } + + let _ = db.cleanup().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + /// Test the [`RegionAllocationStrategy::RandomWithDistinctSleds`] strategy. + /// It should always pick datasets where no two datasets are on the same + /// zpool and no two zpools are on the same sled. + async fn test_region_allocation_strat_random_with_distinct_sleds() { + let logctx = dev::test_setup_log( + "test_region_allocation_strat_random_with_distinct_sleds", + ); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // Create a rack without enough sleds for a successful allocation when + // we require 3 distinct sleds. + let test_datasets = create_test_datasets_for_region_allocation( + &opctx, + datastore.clone(), + REGION_REDUNDANCY_THRESHOLD, + ) + .await; + + // We need to check that our datasets end up on 3 distinct sleds, but the query doesn't return the sled ID, so we need to reverse map from dataset ID to sled ID + let sled_id_map: HashMap = test_datasets + .into_iter() + .map(|test_dataset| (test_dataset.dataset_id, test_dataset.sled_id)) + .collect(); + + // Allocate regions from the datasets for this disk. Do it a few times + // for good measure. + for alloc_seed in 0..10 { + let params = create_test_disk_create_params( + &format!("disk{}", alloc_seed), + ByteCount::from_mebibytes_u32(1), + ); + let volume_id = Uuid::new_v4(); + + let expected_region_count = REGION_REDUNDANCY_THRESHOLD; + let dataset_and_regions = datastore + .region_allocate( + &opctx, + volume_id, + ¶ms.disk_source, + params.size, + &&RegionAllocationStrategy::RandomWithDistinctSleds { + seed: Some(alloc_seed), + }, + ) + .await + .unwrap(); + + // Verify the allocation. + assert_eq!(expected_region_count, dataset_and_regions.len()); + let mut disk_datasets = HashSet::new(); + let mut disk_zpools = HashSet::new(); + let mut disk_sleds = HashSet::new(); for (dataset, region) in dataset_and_regions { // Must be 3 unique datasets assert!(disk_datasets.insert(dataset.id())); @@ -819,8 +886,8 @@ mod test { assert!(disk_zpools.insert(dataset.pool_id)); // Must be 3 unique sleds - // TODO: When allocation chooses 3 distinct sleds, uncomment this. - // assert!(disk1_sleds.insert(Err(dataset))); + let sled_id = sled_id_map.get(&dataset.id()).unwrap(); + assert!(disk_sleds.insert(*sled_id)); assert_eq!(volume_id, region.volume_id()); assert_eq!(ByteCount::from(4096), region.block_size()); @@ -836,14 +903,72 @@ mod test { logctx.cleanup_successful(); } + #[tokio::test] + /// Ensure the [`RegionAllocationStrategy::RandomWithDistinctSleds`] + /// strategy fails when there aren't enough distinct sleds. + async fn test_region_allocation_strat_random_with_distinct_sleds_fails() { + let logctx = dev::test_setup_log( + "test_region_allocation_strat_random_with_distinct_sleds_fails", + ); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // Create a rack without enough sleds for a successful allocation when + // we require 3 distinct sleds. + create_test_datasets_for_region_allocation( + &opctx, + datastore.clone(), + REGION_REDUNDANCY_THRESHOLD - 1, + ) + .await; + + // Allocate regions from the datasets for this disk. Do it a few times + // for good measure. + for alloc_seed in 0..10 { + let params = create_test_disk_create_params( + &format!("disk{}", alloc_seed), + ByteCount::from_mebibytes_u32(1), + ); + let volume_id = Uuid::new_v4(); + + let err = datastore + .region_allocate( + &opctx, + volume_id, + ¶ms.disk_source, + params.size, + &&RegionAllocationStrategy::RandomWithDistinctSleds { + seed: Some(alloc_seed), + }, + ) + .await + .unwrap_err(); + + let expected = "Not enough zpool space to allocate disks"; + assert!( + err.to_string().contains(expected), + "Saw error: \'{err}\', but expected \'{expected}\'" + ); + + assert!(matches!(err, Error::ServiceUnavailable { .. })); + } + + let _ = db.cleanup().await; + logctx.cleanup_successful(); + } + #[tokio::test] async fn test_region_allocation_is_idempotent() { let logctx = dev::test_setup_log("test_region_allocation_is_idempotent"); let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; - create_test_datasets_for_region_allocation(&opctx, datastore.clone()) - .await; + create_test_datasets_for_region_allocation( + &opctx, + datastore.clone(), + REGION_REDUNDANCY_THRESHOLD, + ) + .await; // Allocate regions from the datasets for this volume. let params = create_test_disk_create_params( @@ -857,7 +982,7 @@ mod test { volume_id, ¶ms.disk_source, params.size, - &RegionAllocationStrategy::Random(Some(0)), + &RegionAllocationStrategy::Random { seed: Some(0) }, ) .await .unwrap(); @@ -870,7 +995,7 @@ mod test { volume_id, ¶ms.disk_source, params.size, - &RegionAllocationStrategy::Random(Some(1)), + &RegionAllocationStrategy::Random { seed: Some(1) }, ) .await .unwrap(); @@ -959,7 +1084,7 @@ mod test { volume1_id, ¶ms.disk_source, params.size, - &RegionAllocationStrategy::Random(Some(0)), + &RegionAllocationStrategy::Random { seed: Some(0) }, ) .await .unwrap_err(); @@ -983,8 +1108,12 @@ mod test { let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; - create_test_datasets_for_region_allocation(&opctx, datastore.clone()) - .await; + create_test_datasets_for_region_allocation( + &opctx, + datastore.clone(), + REGION_REDUNDANCY_THRESHOLD, + ) + .await; let disk_size = test_zpool_size(); let alloc_size = ByteCount::try_from(disk_size.to_bytes() * 2).unwrap(); @@ -997,7 +1126,7 @@ mod test { volume1_id, ¶ms.disk_source, params.size, - &RegionAllocationStrategy::Random(Some(0)), + &RegionAllocationStrategy::Random { seed: Some(0) }, ) .await .is_err()); diff --git a/nexus/db-queries/src/db/datastore/region.rs b/nexus/db-queries/src/db/datastore/region.rs index 5bc79b9481..9465fe2792 100644 --- a/nexus/db-queries/src/db/datastore/region.rs +++ b/nexus/db-queries/src/db/datastore/region.rs @@ -5,7 +5,6 @@ //! [`DataStore`] methods on [`Region`]s. use super::DataStore; -use super::RegionAllocationStrategy; use super::RunnableQuery; use crate::context::OpContext; use crate::db; @@ -23,6 +22,7 @@ use omicron_common::api::external; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::backoff::{self, BackoffError}; +use omicron_common::nexus_config::RegionAllocationStrategy; use slog::Logger; use uuid::Uuid; diff --git a/nexus/db-queries/src/db/queries/region_allocation.rs b/nexus/db-queries/src/db/queries/region_allocation.rs index b071ee3f44..7f7b2ea9bf 100644 --- a/nexus/db-queries/src/db/queries/region_allocation.rs +++ b/nexus/db-queries/src/db/queries/region_allocation.rs @@ -6,7 +6,6 @@ use crate::db::alias::ExpressionAlias; use crate::db::cast_uuid_as_bytea::CastUuidToBytea; -use crate::db::datastore::RegionAllocationStrategy; use crate::db::datastore::REGION_REDUNDANCY_THRESHOLD; use crate::db::model::{Dataset, DatasetKind, Region}; use crate::db::pool::DbConnection; @@ -24,10 +23,11 @@ use diesel::{ use nexus_db_model::queries::region_allocation::{ candidate_datasets, candidate_regions, candidate_zpools, cockroach_md5, do_insert, inserted_regions, old_regions, old_zpool_usage, - proposed_dataset_changes, updated_datasets, + proposed_dataset_changes, shuffled_candidate_datasets, updated_datasets, }; use nexus_db_model::schema; use omicron_common::api::external; +use omicron_common::nexus_config::RegionAllocationStrategy; const NOT_ENOUGH_DATASETS_SENTINEL: &'static str = "Not enough datasets"; const NOT_ENOUGH_ZPOOL_SPACE_SENTINEL: &'static str = "Not enough space"; @@ -53,7 +53,7 @@ pub fn from_diesel(e: async_bb8_diesel::ConnectionError) -> external::Error { } NOT_ENOUGH_ZPOOL_SPACE_SENTINEL => { return external::Error::unavail( - "Not enough zpool space to allocate disks", + "Not enough zpool space to allocate disks. There may not be enough disks with space for the requested region. You may also see this if your rack is in a degraded state, or you're running the default multi-rack topology configuration in a 1-sled development environment.", ); } NOT_ENOUGH_UNIQUE_ZPOOLS_SENTINEL => { @@ -91,6 +91,8 @@ impl OldRegions { /// This implicitly distinguishes between "M.2s" and "U.2s" -- Nexus needs to /// determine during dataset provisioning which devices should be considered for /// usage as Crucible storage. +/// +/// We select only one dataset from each zpool. #[derive(Subquery, QueryId)] #[subquery(name = candidate_datasets)] struct CandidateDatasets { @@ -98,71 +100,65 @@ struct CandidateDatasets { } impl CandidateDatasets { - fn new( - allocation_strategy: &RegionAllocationStrategy, - candidate_zpools: &CandidateZpools, - ) -> Self { + fn new(candidate_zpools: &CandidateZpools, seed: u128) -> Self { use crate::db::schema::dataset::dsl as dataset_dsl; use candidate_zpools::dsl as candidate_zpool_dsl; - let query = match allocation_strategy { - #[cfg(test)] - RegionAllocationStrategy::LeastUsedDisk => { - let query: Box< - dyn CteQuery, - > = Box::new( - dataset_dsl::dataset - .inner_join( - candidate_zpools - .query_source() - .on(dataset_dsl::pool_id - .eq(candidate_zpool_dsl::pool_id)), - ) - .filter(dataset_dsl::time_deleted.is_null()) - .filter(dataset_dsl::size_used.is_not_null()) - .filter(dataset_dsl::kind.eq(DatasetKind::Crucible)) - .order(dataset_dsl::size_used.asc()) - .limit(REGION_REDUNDANCY_THRESHOLD.try_into().unwrap()) - .select((dataset_dsl::id, dataset_dsl::pool_id)), - ); - query - } - RegionAllocationStrategy::Random(seed) => { - let seed = seed.unwrap_or_else(|| { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - }); - - let seed_bytes = seed.to_le_bytes(); - - let query: Box< - dyn CteQuery, - > = Box::new( - dataset_dsl::dataset - .inner_join( - candidate_zpools - .query_source() - .on(dataset_dsl::pool_id - .eq(candidate_zpool_dsl::pool_id)), - ) - .filter(dataset_dsl::time_deleted.is_null()) - .filter(dataset_dsl::size_used.is_not_null()) - .filter(dataset_dsl::kind.eq(DatasetKind::Crucible)) - // We order by md5 to shuffle the ordering of the datasets. - // md5 has a uniform output distribution so it does the job. - .order(cockroach_md5::dsl::md5( + let seed_bytes = seed.to_le_bytes(); + + let query: Box> = + Box::new( + dataset_dsl::dataset + .inner_join(candidate_zpools.query_source().on( + dataset_dsl::pool_id.eq(candidate_zpool_dsl::pool_id), + )) + .filter(dataset_dsl::time_deleted.is_null()) + .filter(dataset_dsl::size_used.is_not_null()) + .filter(dataset_dsl::kind.eq(DatasetKind::Crucible)) + .distinct_on(dataset_dsl::pool_id) + .order_by(( + dataset_dsl::pool_id, + cockroach_md5::dsl::md5( CastUuidToBytea::new(dataset_dsl::id) .concat(seed_bytes.to_vec()), - )) - .select((dataset_dsl::id, dataset_dsl::pool_id)) - .limit(REGION_REDUNDANCY_THRESHOLD.try_into().unwrap()), - ); - query - } - }; + ), + )) + .select((dataset_dsl::id, dataset_dsl::pool_id)), + ); + Self { query } + } +} + +/// Shuffle the candidate datasets, and select REGION_REDUNDANCY_THRESHOLD +/// regions from it. +#[derive(Subquery, QueryId)] +#[subquery(name = shuffled_candidate_datasets)] +struct ShuffledCandidateDatasets { + query: Box>, +} +impl ShuffledCandidateDatasets { + fn new(candidate_datasets: &CandidateDatasets, seed: u128) -> Self { + use candidate_datasets::dsl as candidate_datasets_dsl; + + let seed_bytes = seed.to_le_bytes(); + + let query: Box> = + Box::new( + candidate_datasets + .query_source() + // We order by md5 to shuffle the ordering of the datasets. + // md5 has a uniform output distribution so it does the job. + .order(cockroach_md5::dsl::md5( + CastUuidToBytea::new(candidate_datasets_dsl::id) + .concat(seed_bytes.to_vec()), + )) + .select(( + candidate_datasets_dsl::id, + candidate_datasets_dsl::pool_id, + )) + .limit(REGION_REDUNDANCY_THRESHOLD.try_into().unwrap()), + ); Self { query } } } @@ -179,14 +175,14 @@ diesel::sql_function!(fn now() -> Timestamptz); impl CandidateRegions { fn new( - candidate_datasets: &CandidateDatasets, + shuffled_candidate_datasets: &ShuffledCandidateDatasets, volume_id: uuid::Uuid, block_size: u64, blocks_per_extent: u64, extent_count: u64, ) -> Self { - use candidate_datasets::dsl as candidate_datasets_dsl; use schema::region; + use shuffled_candidate_datasets::dsl as shuffled_candidate_datasets_dsl; let volume_id = volume_id.into_sql::(); let block_size = (block_size as i64).into_sql::(); @@ -195,20 +191,22 @@ impl CandidateRegions { let extent_count = (extent_count as i64).into_sql::(); Self { - query: Box::new(candidate_datasets.query_source().select(( - ExpressionAlias::new::(gen_random_uuid()), - ExpressionAlias::new::(now()), - ExpressionAlias::new::(now()), - ExpressionAlias::new::( - candidate_datasets_dsl::id, + query: Box::new(shuffled_candidate_datasets.query_source().select( + ( + ExpressionAlias::new::(gen_random_uuid()), + ExpressionAlias::new::(now()), + ExpressionAlias::new::(now()), + ExpressionAlias::new::( + shuffled_candidate_datasets_dsl::id, + ), + ExpressionAlias::new::(volume_id), + ExpressionAlias::new::(block_size), + ExpressionAlias::new::( + blocks_per_extent, + ), + ExpressionAlias::new::(extent_count), ), - ExpressionAlias::new::(volume_id), - ExpressionAlias::new::(block_size), - ExpressionAlias::new::( - blocks_per_extent, - ), - ExpressionAlias::new::(extent_count), - ))), + )), } } } @@ -285,12 +283,14 @@ struct CandidateZpools { } impl CandidateZpools { - fn new(old_zpool_usage: &OldPoolUsage, zpool_size_delta: u64) -> Self { + fn new( + old_zpool_usage: &OldPoolUsage, + zpool_size_delta: u64, + seed: u128, + distinct_sleds: bool, + ) -> Self { use schema::zpool::dsl as zpool_dsl; - let with_zpool = zpool_dsl::zpool - .on(zpool_dsl::id.eq(old_zpool_usage::dsl::pool_id)); - // Why are we using raw `diesel::dsl::sql` here? // // When SQL performs the "SUM" operation on "bigint" type, the result @@ -309,15 +309,40 @@ impl CandidateZpools { + diesel::dsl::sql(&zpool_size_delta.to_string())) .le(diesel::dsl::sql(zpool_dsl::total_size::NAME)); - Self { - query: Box::new( - old_zpool_usage - .query_source() - .inner_join(with_zpool) - .filter(it_will_fit) - .select((old_zpool_usage::dsl::pool_id,)), - ), - } + let with_zpool = zpool_dsl::zpool + .on(zpool_dsl::id.eq(old_zpool_usage::dsl::pool_id)); + + let base_query = old_zpool_usage + .query_source() + .inner_join(with_zpool) + .filter(it_will_fit) + .select((old_zpool_usage::dsl::pool_id,)); + + let query = if distinct_sleds { + let seed_bytes = seed.to_le_bytes(); + + let query: Box> = + Box::new( + base_query + .order_by(( + zpool_dsl::sled_id, + cockroach_md5::dsl::md5( + CastUuidToBytea::new(zpool_dsl::id) + .concat(seed_bytes.to_vec()), + ), + )) + .distinct_on(zpool_dsl::sled_id), + ); + + query + } else { + let query: Box> = + Box::new(base_query); + + query + }; + + Self { query } } } @@ -508,19 +533,47 @@ impl RegionAllocate { extent_count: u64, allocation_strategy: &RegionAllocationStrategy, ) -> Self { + let (seed, distinct_sleds) = { + let (input_seed, distinct_sleds) = match allocation_strategy { + RegionAllocationStrategy::Random { seed } => (seed, false), + RegionAllocationStrategy::RandomWithDistinctSleds { seed } => { + (seed, true) + } + }; + ( + input_seed.map_or_else( + || { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + }, + |seed| seed as u128, + ), + distinct_sleds, + ) + }; + let size_delta = block_size * blocks_per_extent * extent_count; let old_regions = OldRegions::new(volume_id); let old_pool_usage = OldPoolUsage::new(); - let candidate_zpools = - CandidateZpools::new(&old_pool_usage, size_delta); + let candidate_zpools = CandidateZpools::new( + &old_pool_usage, + size_delta, + seed, + distinct_sleds, + ); let candidate_datasets = - CandidateDatasets::new(&allocation_strategy, &candidate_zpools); + CandidateDatasets::new(&candidate_zpools, seed); + + let shuffled_candidate_datasets = + ShuffledCandidateDatasets::new(&candidate_datasets, seed); let candidate_regions = CandidateRegions::new( - &candidate_datasets, + &shuffled_candidate_datasets, volume_id, block_size, blocks_per_extent, @@ -577,6 +630,7 @@ impl RegionAllocate { .add_subquery(old_pool_usage) .add_subquery(candidate_zpools) .add_subquery(candidate_datasets) + .add_subquery(shuffled_candidate_datasets) .add_subquery(candidate_regions) .add_subquery(proposed_changes) .add_subquery(do_insert) diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index f1b20c32a1..1a9afbc6bd 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -92,3 +92,14 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 + +[default_region_allocation_strategy] +# allocate region on 3 random distinct zpools, on 3 random distinct sleds. +type = "random_with_distinct_sleds" + +# the same as random_with_distinct_sleds, but without requiring distinct sleds +# type = "random" + +# setting `seed` to a fixed value will make dataset selection ordering use the +# same shuffling order for every region allocation. +# seed = 0 diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 5bab5e2820..354df0ead3 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -23,6 +23,7 @@ use omicron_common::address::DENDRITE_PORT; use omicron_common::address::MGS_PORT; use omicron_common::api::external::Error; use omicron_common::api::internal::shared::SwitchLocation; +use omicron_common::nexus_config::RegionAllocationStrategy; use slog::Logger; use std::collections::HashMap; use std::net::Ipv6Addr; @@ -153,6 +154,9 @@ pub struct Nexus { /// Background tasks background_tasks: background::BackgroundTasks, + + /// Default Crucible region allocation strategy + default_region_allocation_strategy: RegionAllocationStrategy, } impl Nexus { @@ -325,6 +329,10 @@ impl Nexus { external_resolver, dpd_clients, background_tasks, + default_region_allocation_strategy: config + .pkg + .default_region_allocation_strategy + .clone(), }; // TODO-cleanup all the extra Arcs here seems wrong diff --git a/nexus/src/app/sagas/disk_create.rs b/nexus/src/app/sagas/disk_create.rs index cca36cefa7..275c8738cc 100644 --- a/nexus/src/app/sagas/disk_create.rs +++ b/nexus/src/app/sagas/disk_create.rs @@ -12,11 +12,10 @@ use super::{ ACTION_GENERATE_ID, }; use crate::app::sagas::declare_saga_actions; +use crate::app::{authn, authz, db}; use crate::external_api::params; -use nexus_db_queries::db::datastore::RegionAllocationStrategy; use nexus_db_queries::db::identity::{Asset, Resource}; use nexus_db_queries::db::lookup::LookupPath; -use nexus_db_queries::{authn, authz, db}; use omicron_common::api::external::DiskState; use omicron_common::api::external::Error; use rand::{rngs::StdRng, RngCore, SeedableRng}; @@ -255,6 +254,9 @@ async fn sdc_alloc_regions( &sagactx, ¶ms.serialized_authn, ); + + let strategy = &osagactx.nexus().default_region_allocation_strategy; + let datasets_and_regions = osagactx .datastore() .region_allocate( @@ -262,7 +264,7 @@ async fn sdc_alloc_regions( volume_id, ¶ms.create_params.disk_source, params.create_params.size, - &RegionAllocationStrategy::Random(None), + &strategy, ) .await .map_err(ActionError::action_failed)?; diff --git a/nexus/src/app/sagas/snapshot_create.rs b/nexus/src/app/sagas/snapshot_create.rs index b27f4a3a9b..eeabf64894 100644 --- a/nexus/src/app/sagas/snapshot_create.rs +++ b/nexus/src/app/sagas/snapshot_create.rs @@ -100,14 +100,13 @@ use super::{ }; use crate::app::sagas::declare_saga_actions; use crate::app::sagas::retry_until_known_result; +use crate::app::{authn, authz, db}; use crate::external_api::params; use anyhow::anyhow; use crucible_agent_client::{types::RegionId, Client as CrucibleAgentClient}; use nexus_db_model::Generation; -use nexus_db_queries::db::datastore::RegionAllocationStrategy; use nexus_db_queries::db::identity::{Asset, Resource}; use nexus_db_queries::db::lookup::LookupPath; -use nexus_db_queries::{authn, authz, db}; use omicron_common::api::external; use omicron_common::api::external::Error; use rand::{rngs::StdRng, RngCore, SeedableRng}; @@ -332,6 +331,8 @@ async fn ssc_alloc_regions( .await .map_err(ActionError::action_failed)?; + let strategy = &osagactx.nexus().default_region_allocation_strategy; + let datasets_and_regions = osagactx .datastore() .region_allocate( @@ -344,7 +345,7 @@ async fn ssc_alloc_regions( .map_err(|e| ActionError::action_failed(e.to_string()))?, }, external::ByteCount::from(disk.size), - &RegionAllocationStrategy::Random(None), + &strategy, ) .await .map_err(ActionError::action_failed)?; diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 6eeacceaed..1b1ae2c912 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -89,3 +89,8 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 + +[default_region_allocation_strategy] +# we only have one sled in the test environment, so we need to use the +# `Random` strategy, instead of `RandomWithDistinctSleds` +type = "random" \ No newline at end of file diff --git a/package-manifest.toml b/package-manifest.toml index 4dc0f6b616..c776f6d96d 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -92,7 +92,8 @@ source.rust.binary_names = ["nexus", "schema-updater"] source.rust.release = true source.paths = [ { from = "/opt/ooce/pgsql-13/lib/amd64", to = "/opt/ooce/pgsql-13/lib/amd64" }, - { from = "smf/nexus", to = "/var/svc/manifest/site/nexus" }, + { from = "smf/nexus/manifest.xml", to = "/var/svc/manifest/site/nexus/manifest.xml" }, + { from = "smf/nexus/{{rack-topology}}", to = "/var/svc/manifest/site/nexus" }, { from = "out/console-assets", to = "/var/nexus/static" }, { from = "schema/crdb", to = "/var/nexus/schema/crdb" }, ] diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index ea490e54cf..bc07b61234 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -211,11 +211,12 @@ async fn do_target( format!("failed to create directory {}", target_dir.display()) })?; match subcommand { - TargetCommand::Create { image, machine, switch } => { + TargetCommand::Create { image, machine, switch, rack_topology } => { let target = KnownTarget::new( image.clone(), machine.clone(), switch.clone(), + rack_topology.clone(), )?; let path = get_single_target(&target_dir, name).await?; diff --git a/package/src/lib.rs b/package/src/lib.rs index b0cc04970a..395f3ed472 100644 --- a/package/src/lib.rs +++ b/package/src/lib.rs @@ -46,6 +46,29 @@ pub enum TargetCommand { #[clap(short, long, default_value_if("image", "standard", "stub"))] switch: Option, + + #[clap( + short, + long, + default_value_if("image", "trampoline", Some("single-sled")), + + // This opt is required, and clap will enforce that even with + // `required = false`, since it's not an Option. But the + // default_value_if only works if we set `required` to false. It's + // jank, but it is what it is. + // https://github.com/clap-rs/clap/issues/4086 + required = false + )] + /// Specify whether nexus will run in a single-sled or multi-sled + /// environment. + /// + /// Set single-sled for dev purposes when you're running a single + /// sled-agent. Set multi-sled if you're running with mulitple sleds. + /// Currently this only affects the crucible disk allocation strategy- + /// VM disks will require 3 distinct sleds with `multi-sled`, which will + /// fail in a single-sled environment. `single-sled` relaxes this + /// requirement. + rack_topology: crate::target::RackTopology, }, /// List all existing targets List, diff --git a/package/src/target.rs b/package/src/target.rs index a7b2dd4539..d5d5e92c46 100644 --- a/package/src/target.rs +++ b/package/src/target.rs @@ -48,12 +48,27 @@ pub enum Switch { SoftNpu, } +/// Topology of the sleds within the rack. +#[derive(Clone, Debug, strum::EnumString, strum::Display, ValueEnum)] +#[strum(serialize_all = "kebab-case")] +#[clap(rename_all = "kebab-case")] +pub enum RackTopology { + /// Use configurations suitable for a multi-sled deployment, such as dogfood + /// and production racks. + MultiSled, + + /// Use configurations suitable for a single-sled deployment, such as CI and + /// dev machines. + SingleSled, +} + /// A strongly-typed variant of [Target]. #[derive(Clone, Debug)] pub struct KnownTarget { image: Image, machine: Option, switch: Option, + rack_topology: RackTopology, } impl KnownTarget { @@ -61,6 +76,7 @@ impl KnownTarget { image: Image, machine: Option, switch: Option, + rack_topology: RackTopology, ) -> Result { if matches!(image, Image::Trampoline) { if machine.is_some() { @@ -77,7 +93,7 @@ impl KnownTarget { bail!("'switch=asic' is only valid with 'machine=gimlet'"); } - Ok(Self { image, machine, switch }) + Ok(Self { image, machine, switch, rack_topology }) } } @@ -87,6 +103,7 @@ impl Default for KnownTarget { image: Image::Standard, machine: Some(Machine::NonGimlet), switch: Some(Switch::Stub), + rack_topology: RackTopology::MultiSled, } } } @@ -101,6 +118,7 @@ impl From for Target { if let Some(switch) = kt.switch { map.insert("switch".to_string(), switch.to_string()); } + map.insert("rack-topology".to_string(), kt.rack_topology.to_string()); Target(map) } } @@ -121,6 +139,7 @@ impl std::str::FromStr for KnownTarget { let mut image = Self::default().image; let mut machine = None; let mut switch = None; + let mut rack_topology = None; for (k, v) in target.0.into_iter() { match k.as_str() { @@ -133,6 +152,9 @@ impl std::str::FromStr for KnownTarget { "switch" => { switch = Some(v.parse()?); } + "rack-topology" => { + rack_topology = Some(v.parse()?); + } _ => { bail!( "Unknown target key {k}\nValid keys include: [{}]", @@ -146,6 +168,11 @@ impl std::str::FromStr for KnownTarget { } } } - KnownTarget::new(image, machine, switch) + KnownTarget::new( + image, + machine, + switch, + rack_topology.unwrap_or(RackTopology::MultiSled), + ) } } diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 88e51a3bc3..d4ccfc97c8 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -120,3 +120,5 @@ machine-non-gimlet = [] switch-asic = [] switch-stub = [] switch-softnpu = [] +rack-topology-single-sled = [] +rack-topology-multi-sled = [] diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 96cdf8222b..60f0965612 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1513,6 +1513,9 @@ impl ServiceManager { .open(&config_path) .await .map_err(|err| Error::io_path(&config_path, err))?; + file.write_all(b"\n\n") + .await + .map_err(|err| Error::io_path(&config_path, err))?; file.write_all(config_str.as_bytes()) .await .map_err(|err| Error::io_path(&config_path, err))?; diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml new file mode 100644 index 0000000000..2dfee81d02 --- /dev/null +++ b/smf/nexus/multi-sled/config-partial.toml @@ -0,0 +1,45 @@ +# +# Oxide API: partial configuration file +# + +[console] +# Directory for static assets. Absolute path or relative to CWD. +static_dir = "/var/nexus/static" +session_idle_timeout_minutes = 60 +session_absolute_timeout_minutes = 480 + +[authn] +schemes_external = ["session_cookie", "access_token"] + +[log] +# Show log messages of this level and more severe +level = "debug" +mode = "file" +path = "/dev/stdout" +if_exists = "append" + +# TODO: Uncomment the following lines to enable automatic schema +# migration on boot. +# +# [schema] +# schema_dir = "/var/nexus/schema/crdb" + +[background_tasks] +dns_internal.period_secs_config = 60 +dns_internal.period_secs_servers = 60 +dns_internal.period_secs_propagation = 60 +dns_internal.max_concurrent_server_updates = 5 +dns_external.period_secs_config = 60 +dns_external.period_secs_servers = 60 +dns_external.period_secs_propagation = 60 +dns_external.max_concurrent_server_updates = 5 +# How frequently we check the list of stored TLS certificates. This is +# approximately an upper bound on how soon after updating the list of +# certificates it will take _other_ Nexus instances to notice and stop serving +# them (on a sunny day). +external_endpoints.period_secs = 60 + +[default_region_allocation_strategy] +# by default, allocate across 3 distinct sleds +# seed is omitted so a new seed will be chosen with every allocation. +type = "random_with_distinct_sleds" \ No newline at end of file diff --git a/smf/nexus/config-partial.toml b/smf/nexus/single-sled/config-partial.toml similarity index 86% rename from smf/nexus/config-partial.toml rename to smf/nexus/single-sled/config-partial.toml index b29727c4aa..aff0a8a25f 100644 --- a/smf/nexus/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -38,3 +38,8 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 + +[default_region_allocation_strategy] +# by default, allocate without requirement for distinct sleds. +# seed is omitted so a new seed will be chosen with every allocation. +type = "random" \ No newline at end of file From d300fb89fb798d4b9cc6b785829ddceffa66ecd4 Mon Sep 17 00:00:00 2001 From: Ryan Goodfellow Date: Tue, 3 Oct 2023 12:45:17 -0700 Subject: [PATCH 14/89] Omdb networking (#4147) --- Cargo.lock | 1 + dev-tools/omdb/Cargo.toml | 1 + dev-tools/omdb/src/bin/omdb/db.rs | 180 ++++++++++++++++++++++++ dev-tools/omdb/tests/test_all_output.rs | 1 + dev-tools/omdb/tests/usage_errors.out | 20 +++ 5 files changed, 203 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 3a45dcb381..c4385bf694 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5127,6 +5127,7 @@ dependencies = [ "expectorate", "humantime", "internal-dns 0.1.0", + "ipnetwork", "nexus-client 0.1.0", "nexus-db-model", "nexus-db-queries", diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml index 5b2adde1b2..5a05e93db9 100644 --- a/dev-tools/omdb/Cargo.toml +++ b/dev-tools/omdb/Cargo.toml @@ -33,6 +33,7 @@ textwrap.workspace = true tokio = { workspace = true, features = [ "full" ] } uuid.workspace = true omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +ipnetwork.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 93e5ef4301..10e5546b6d 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -12,6 +12,9 @@ //! would be the only consumer -- and in that case it's okay to query the //! database directly. +// NOTE: eminates from Tabled macros +#![allow(clippy::useless_vec)] + use crate::Omdb; use anyhow::anyhow; use anyhow::bail; @@ -30,7 +33,9 @@ use nexus_db_model::DnsGroup; use nexus_db_model::DnsName; use nexus_db_model::DnsVersion; use nexus_db_model::DnsZone; +use nexus_db_model::ExternalIp; use nexus_db_model::Instance; +use nexus_db_model::Project; use nexus_db_model::Region; use nexus_db_model::Sled; use nexus_db_model::Zpool; @@ -86,6 +91,8 @@ enum DbCommands { Sleds, /// Print information about customer instances Instances, + /// Print information about the network + Network(NetworkArgs), } #[derive(Debug, Args)] @@ -170,6 +177,22 @@ enum ServicesCommands { ListBySled, } +#[derive(Debug, Args)] +struct NetworkArgs { + #[command(subcommand)] + command: NetworkCommands, + + /// Print out raw data structures from the data store. + #[clap(long)] + verbose: bool, +} + +#[derive(Debug, Subcommand)] +enum NetworkCommands { + /// List external IPs + ListEips, +} + impl DbArgs { /// Run a `omdb db` subcommand. pub(crate) async fn run_cmd( @@ -269,6 +292,13 @@ impl DbArgs { DbCommands::Instances => { cmd_db_instances(&datastore, self.fetch_limit).await } + DbCommands::Network(NetworkArgs { + command: NetworkCommands::ListEips, + verbose, + }) => { + cmd_db_eips(&opctx, &datastore, self.fetch_limit, *verbose) + .await + } } } } @@ -1098,6 +1128,156 @@ async fn cmd_db_dns_names( Ok(()) } +async fn cmd_db_eips( + opctx: &OpContext, + datastore: &DataStore, + limit: NonZeroU32, + verbose: bool, +) -> Result<(), anyhow::Error> { + use db::schema::external_ip::dsl; + let ips: Vec = dsl::external_ip + .filter(dsl::time_deleted.is_null()) + .select(ExternalIp::as_select()) + .get_results_async(&*datastore.pool_connection_for_tests().await?) + .await?; + + check_limit(&ips, limit, || String::from("listing external ips")); + + struct PortRange { + first: u16, + last: u16, + } + + impl Display for PortRange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}/{}", self.first, self.last) + } + } + + #[derive(Tabled)] + enum Owner { + Instance { project: String, name: String }, + Service { kind: String }, + None, + } + + impl Display for Owner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Instance { project, name } => { + write!(f, "Instance {project}/{name}") + } + Self::Service { kind } => write!(f, "Service {kind}"), + Self::None => write!(f, "None"), + } + } + } + + #[derive(Tabled)] + struct IpRow { + ip: ipnetwork::IpNetwork, + ports: PortRange, + kind: String, + owner: Owner, + } + + if verbose { + for ip in &ips { + if verbose { + println!("{ip:#?}"); + } + } + return Ok(()); + } + + let mut rows = Vec::new(); + + for ip in &ips { + let owner = if let Some(owner_id) = ip.parent_id { + if ip.is_service { + let service = match LookupPath::new(opctx, datastore) + .service_id(owner_id) + .fetch() + .await + { + Ok(instance) => instance, + Err(e) => { + eprintln!( + "error looking up service with id {owner_id}: {e}" + ); + continue; + } + }; + Owner::Service { kind: format!("{:?}", service.1.kind) } + } else { + use db::schema::instance::dsl as instance_dsl; + let instance = match instance_dsl::instance + .filter(instance_dsl::id.eq(owner_id)) + .limit(1) + .select(Instance::as_select()) + .load_async(&*datastore.pool_connection_for_tests().await?) + .await + .context("loading requested instance")? + .pop() + { + Some(instance) => instance, + None => { + eprintln!("instance with id {owner_id} not found"); + continue; + } + }; + + use db::schema::project::dsl as project_dsl; + let project = match project_dsl::project + .filter(project_dsl::id.eq(instance.project_id)) + .limit(1) + .select(Project::as_select()) + .load_async(&*datastore.pool_connection_for_tests().await?) + .await + .context("loading requested project")? + .pop() + { + Some(instance) => instance, + None => { + eprintln!( + "project with id {} not found", + instance.project_id + ); + continue; + } + }; + + Owner::Instance { + project: project.name().to_string(), + name: instance.name().to_string(), + } + } + } else { + Owner::None + }; + + let row = IpRow { + ip: ip.ip, + ports: PortRange { + first: ip.first_port.into(), + last: ip.last_port.into(), + }, + kind: format!("{:?}", ip.kind), + owner, + }; + rows.push(row); + } + + rows.sort_by(|a, b| a.ip.cmp(&b.ip)); + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .to_string(); + + println!("{}", table); + + Ok(()) +} + fn print_name( prefix: &str, name: &str, diff --git a/dev-tools/omdb/tests/test_all_output.rs b/dev-tools/omdb/tests/test_all_output.rs index 0eddcb492c..d757369ead 100644 --- a/dev-tools/omdb/tests/test_all_output.rs +++ b/dev-tools/omdb/tests/test_all_output.rs @@ -41,6 +41,7 @@ async fn test_omdb_usage_errors() { &["db", "dns", "diff"], &["db", "dns", "names"], &["db", "services"], + &["db", "network"], &["nexus"], &["nexus", "background-tasks"], &["sled-agent"], diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index 136a631e80..b5421b76af 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -91,6 +91,7 @@ Commands: services Print information about control plane services sleds Print information about sleds instances Print information about customer instances + network Print information about the network help Print this message or the help of the given subcommand(s) Options: @@ -112,6 +113,7 @@ Commands: services Print information about control plane services sleds Print information about sleds instances Print information about customer instances + network Print information about the network help Print this message or the help of the given subcommand(s) Options: @@ -186,6 +188,24 @@ Commands: Options: -h, --help Print help ============================================= +EXECUTING COMMAND: omdb ["db", "network"] +termination: Exited(2) +--------------------------------------------- +stdout: +--------------------------------------------- +stderr: +Print information about the network + +Usage: omdb db network [OPTIONS] + +Commands: + list-eips List external IPs + help Print this message or the help of the given subcommand(s) + +Options: + --verbose Print out raw data structures from the data store + -h, --help Print help +============================================= EXECUTING COMMAND: omdb ["nexus"] termination: Exited(2) --------------------------------------------- From 3a5a7cd9d57bd2ed5f2e71d9967eb0b40a236d98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:43:16 -0700 Subject: [PATCH 15/89] Bump thiserror from 1.0.48 to 1.0.49 (#4173) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.48 to 1.0.49.
Release notes

Sourced from thiserror's releases.

1.0.49

  • Access libcore types through ::core in generated code (#255, thanks @​mina86)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=thiserror&package-manager=cargo&previous-version=1.0.48&new-version=1.0.49)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c4385bf694..6133dabc0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8755,18 +8755,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", From af14d1abb5d1fc684157a560d7e0a3cd6a2f860b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:43:28 -0700 Subject: [PATCH 16/89] Bump expectorate from 1.0.7 to 1.1.0 (#4176) Bumps [expectorate](https://github.com/oxidecomputer/expectorate) from 1.0.7 to 1.1.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=expectorate&package-manager=cargo&previous-version=1.0.7&new-version=1.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6133dabc0a..54e74ce07a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2404,9 +2404,9 @@ dependencies = [ [[package]] name = "expectorate" -version = "1.0.7" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "710ab6a2d57038a835d66f78d5af3fa5d27c1ec4682f823b9203c48826cb0591" +checksum = "de6f19b25bdfa2747ae775f37cd109c31f1272d4e4c83095be0727840aa1d75f" dependencies = [ "console", "newline-converter", diff --git a/Cargo.toml b/Cargo.toml index 63d8e0b2d6..e453c47244 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -180,7 +180,7 @@ dns-service-client = { path = "dns-service-client" } dpd-client = { path = "dpd-client" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } either = "1.9.0" -expectorate = "1.0.7" +expectorate = "1.1.0" fatfs = "0.3.6" flate2 = "1.0.27" flume = "0.11.0" From 901d005ebf913be82b50ce6e66de9775c06a954e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 22:44:48 -0700 Subject: [PATCH 17/89] Bump hubtools from `0c642f6` to `2481445` (#4178) Bumps [hubtools](https://github.com/oxidecomputer/hubtools) from `0c642f6` to `2481445`.
Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54e74ce07a..c6335cb32e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3219,7 +3219,7 @@ dependencies = [ [[package]] name = "hubtools" version = "0.4.1" -source = "git+https://github.com/oxidecomputer/hubtools.git?branch=main#0c642f6e1f83b74725c7119a546bc26ac7452a48" +source = "git+https://github.com/oxidecomputer/hubtools.git?branch=main#2481445b80f8476041f62a1c8b6301e4918c63ed" dependencies = [ "lpc55_areas", "lpc55_sign", @@ -4008,7 +4008,7 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lpc55_areas" version = "0.2.4" -source = "git+https://github.com/oxidecomputer/lpc55_support#4051a3b9421573dc36ed6098b292a7609a3cf98b" +source = "git+https://github.com/oxidecomputer/lpc55_support#96f064eaae5e95930efaab6c29fd1b2e22225dac" dependencies = [ "bitfield", "clap 4.4.3", @@ -4018,8 +4018,8 @@ dependencies = [ [[package]] name = "lpc55_sign" -version = "0.3.2" -source = "git+https://github.com/oxidecomputer/lpc55_support#4051a3b9421573dc36ed6098b292a7609a3cf98b" +version = "0.3.3" +source = "git+https://github.com/oxidecomputer/lpc55_support#96f064eaae5e95930efaab6c29fd1b2e22225dac" dependencies = [ "byteorder", "const-oid", @@ -9429,8 +9429,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 1.0.0", - "rand 0.8.5", + "cfg-if 0.1.10", + "rand 0.4.6", "static_assertions", ] From a9104a0786f437e38d4945dd77470bc8fd4f974c Mon Sep 17 00:00:00 2001 From: Rain Date: Wed, 4 Oct 2023 00:12:30 -0700 Subject: [PATCH 18/89] [workspace-hack] use workspace-dotted format and a patch directive (#4197) Two changes: 1. Switch to the workspace-dotted format (`.workspace = true`) for uniformity with the rest of omicron. This is new in cargo-hakari 0.9.28. 2. Use a patch directive, which means that the workspace-hack only applies while building within this workspace. If another workspace imports a crate from here via a git dependency, it will not have the workspace-hack applied to it (instead, it will use [this empty crate](https://crates.io/crates/omicron-workspace-hack) on crates.io). Thanks so much to @pfmooney for this suggestion! Also remove one of the exceptions made in the xtask (workspace-hack lines in *other* `Cargo.toml`s are now output as `.workspace = true`, but hakari cannot yet generate workspace lines in its own `Cargo.toml`). I verified by creating an empty project that the workspace-hack isn't applied to downstream projects that import e.g. `omicron-common` as a git (or path) dependency. Folks will have to update to cargo-hakari 0.9.28 to use this, but hopefully that won't be too much of a bother. --- .config/hakari.toml | 5 ++++- Cargo.toml | 8 ++++++++ api_identity/Cargo.toml | 2 +- bootstore/Cargo.toml | 2 +- bootstrap-agent-client/Cargo.toml | 2 +- caboose-util/Cargo.toml | 2 +- certificates/Cargo.toml | 2 +- common/Cargo.toml | 2 +- crdb-seed/Cargo.toml | 2 +- ddm-admin-client/Cargo.toml | 2 +- deploy/Cargo.toml | 2 +- dev-tools/omdb/Cargo.toml | 2 +- dev-tools/omicron-dev/Cargo.toml | 2 +- dev-tools/xtask/src/main.rs | 6 ------ dns-server/Cargo.toml | 2 +- dns-service-client/Cargo.toml | 2 +- dpd-client/Cargo.toml | 2 +- end-to-end-tests/Cargo.toml | 2 +- gateway-cli/Cargo.toml | 2 +- gateway-client/Cargo.toml | 2 +- gateway-test-utils/Cargo.toml | 2 +- gateway/Cargo.toml | 2 +- illumos-utils/Cargo.toml | 2 +- installinator-artifact-client/Cargo.toml | 2 +- installinator-artifactd/Cargo.toml | 2 +- installinator-common/Cargo.toml | 2 +- installinator/Cargo.toml | 4 ++-- internal-dns-cli/Cargo.toml | 2 +- internal-dns/Cargo.toml | 2 +- ipcc-key-value/Cargo.toml | 2 +- key-manager/Cargo.toml | 2 +- nexus-client/Cargo.toml | 2 +- nexus/Cargo.toml | 2 +- nexus/authz-macros/Cargo.toml | 2 +- nexus/db-macros/Cargo.toml | 2 +- nexus/db-model/Cargo.toml | 2 +- nexus/db-queries/Cargo.toml | 2 +- nexus/defaults/Cargo.toml | 2 +- nexus/test-interface/Cargo.toml | 2 +- nexus/test-utils-macros/Cargo.toml | 2 +- nexus/test-utils/Cargo.toml | 2 +- nexus/types/Cargo.toml | 2 +- oxide-client/Cargo.toml | 2 +- oximeter-client/Cargo.toml | 2 +- oximeter/collector/Cargo.toml | 2 +- oximeter/db/Cargo.toml | 2 +- oximeter/instruments/Cargo.toml | 2 +- oximeter/oximeter-macro-impl/Cargo.toml | 2 +- oximeter/oximeter/Cargo.toml | 2 +- oximeter/producer/Cargo.toml | 2 +- package/Cargo.toml | 2 +- passwords/Cargo.toml | 2 +- rpaths/Cargo.toml | 2 +- sled-agent-client/Cargo.toml | 2 +- sled-agent/Cargo.toml | 2 +- sled-hardware/Cargo.toml | 2 +- sp-sim/Cargo.toml | 2 +- test-utils/Cargo.toml | 2 +- tufaceous-lib/Cargo.toml | 2 +- tufaceous/Cargo.toml | 2 +- update-engine/Cargo.toml | 2 +- wicket-common/Cargo.toml | 2 +- wicket-dbg/Cargo.toml | 2 +- wicket/Cargo.toml | 2 +- wicketd-client/Cargo.toml | 2 +- wicketd/Cargo.toml | 2 +- 66 files changed, 76 insertions(+), 71 deletions(-) diff --git a/.config/hakari.toml b/.config/hakari.toml index 62f15df276..0d883dc6f6 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -6,6 +6,10 @@ hakari-package = "omicron-workspace-hack" # Format for `workspace-hack = ...` lines in other Cargo.tomls. Requires cargo-hakari 0.9.8 or above. dep-format-version = "4" +# Output lines as `omicron-workspace-hack.workspace = true`. Requires +# cargo-hakari 0.9.28 or above. +workspace-hack-line-style = "workspace-dotted" + # Setting workspace.resolver = "2" in the root Cargo.toml is HIGHLY recommended. # Hakari works much better with the new feature resolver. # For more about the new feature resolver, see: @@ -27,4 +31,3 @@ exact-versions = true [traversal-excludes] workspace-members = ["xtask"] - diff --git a/Cargo.toml b/Cargo.toml index e453c47244..fb610128ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -229,6 +229,7 @@ nexus-db-queries = { path = "nexus/db-queries" } nexus-defaults = { path = "nexus/defaults" } omicron-certificates = { path = "certificates" } omicron-passwords = { path = "passwords" } +omicron-workspace-hack = "0.1.0" nexus-test-interface = { path = "nexus/test-interface" } nexus-test-utils-macros = { path = "nexus/test-utils-macros" } nexus-test-utils = { path = "nexus/test-utils" } @@ -554,3 +555,10 @@ opt-level = 3 [patch.crates-io.pq-sys] git = 'https://github.com/oxidecomputer/pq-sys' branch = "oxide/omicron" + +# Using the workspace-hack via this patch directive means that it only applies +# while building within this workspace. If another workspace imports a crate +# from here via a git dependency, it will not have the workspace-hack applied +# to it. +[patch.crates-io.omicron-workspace-hack] +path = "workspace-hack" diff --git a/api_identity/Cargo.toml b/api_identity/Cargo.toml index 9faf2a1878..547defa7c5 100644 --- a/api_identity/Cargo.toml +++ b/api_identity/Cargo.toml @@ -14,4 +14,4 @@ proc-macro = true proc-macro2.workspace = true quote.workspace = true syn.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/bootstore/Cargo.toml b/bootstore/Cargo.toml index eefe05c8d6..18e3e3876b 100644 --- a/bootstore/Cargo.toml +++ b/bootstore/Cargo.toml @@ -36,7 +36,7 @@ zeroize.workspace = true # utils`. Unfortunately, it doesn't appear possible to put the `pq-sys` dep # only in `[dev-dependencies]`. pq-sys = "*" -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] assert_matches.workspace = true diff --git a/bootstrap-agent-client/Cargo.toml b/bootstrap-agent-client/Cargo.toml index 17989a5c5f..42ae59b7aa 100644 --- a/bootstrap-agent-client/Cargo.toml +++ b/bootstrap-agent-client/Cargo.toml @@ -17,4 +17,4 @@ serde.workspace = true sled-hardware.workspace = true slog.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/caboose-util/Cargo.toml b/caboose-util/Cargo.toml index 253d54643d..91bf00741e 100644 --- a/caboose-util/Cargo.toml +++ b/caboose-util/Cargo.toml @@ -7,4 +7,4 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true hubtools.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/certificates/Cargo.toml b/certificates/Cargo.toml index d20d257e4c..87b12fd167 100644 --- a/certificates/Cargo.toml +++ b/certificates/Cargo.toml @@ -12,7 +12,7 @@ openssl-sys.workspace = true thiserror.workspace = true omicron-common.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] omicron-test-utils.workspace = true diff --git a/common/Cargo.toml b/common/Cargo.toml index bda88d0d43..75c1efab55 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -40,7 +40,7 @@ toml.workspace = true uuid.workspace = true parse-display.workspace = true progenitor.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] camino-tempfile.workspace = true diff --git a/crdb-seed/Cargo.toml b/crdb-seed/Cargo.toml index fa71fe7e8a..8d6d570d08 100644 --- a/crdb-seed/Cargo.toml +++ b/crdb-seed/Cargo.toml @@ -13,4 +13,4 @@ omicron-test-utils.workspace = true ring.workspace = true slog.workspace = true tokio.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/ddm-admin-client/Cargo.toml b/ddm-admin-client/Cargo.toml index 3814446b3e..4d00f329e7 100644 --- a/ddm-admin-client/Cargo.toml +++ b/ddm-admin-client/Cargo.toml @@ -15,7 +15,7 @@ tokio.workspace = true omicron-common.workspace = true sled-hardware.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [build-dependencies] anyhow.workspace = true diff --git a/deploy/Cargo.toml b/deploy/Cargo.toml index 17bacd6354..1a6c05a546 100644 --- a/deploy/Cargo.toml +++ b/deploy/Cargo.toml @@ -14,7 +14,7 @@ serde.workspace = true serde_derive.workspace = true thiserror.workspace = true toml.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [[bin]] name = "thing-flinger" diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml index 5a05e93db9..f865acff2b 100644 --- a/dev-tools/omdb/Cargo.toml +++ b/dev-tools/omdb/Cargo.toml @@ -32,8 +32,8 @@ tabled.workspace = true textwrap.workspace = true tokio = { workspace = true, features = [ "full" ] } uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } ipnetwork.workspace = true +omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml index 95da4d42ef..5439b69c76 100644 --- a/dev-tools/omicron-dev/Cargo.toml +++ b/dev-tools/omicron-dev/Cargo.toml @@ -28,7 +28,7 @@ signal-hook-tokio.workspace = true tokio = { workspace = true, features = [ "full" ] } tokio-postgres.workspace = true toml.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] camino-tempfile.workspace = true diff --git a/dev-tools/xtask/src/main.rs b/dev-tools/xtask/src/main.rs index 3e52d742f5..93d91799bc 100644 --- a/dev-tools/xtask/src/main.rs +++ b/dev-tools/xtask/src/main.rs @@ -133,12 +133,6 @@ fn cmd_check_workspace_deps() -> Result<()> { } } - if name == WORKSPACE_HACK_PACKAGE_NAME { - // Skip over workspace-hack because hakari doesn't yet support - // workspace deps: https://github.com/guppy-rs/guppy/issues/7 - continue; - } - non_workspace_dependencies .entry(name.to_owned()) .or_insert_with(Vec::new) diff --git a/dns-server/Cargo.toml b/dns-server/Cargo.toml index d7606dcff5..f91cbfafdb 100644 --- a/dns-server/Cargo.toml +++ b/dns-server/Cargo.toml @@ -30,7 +30,7 @@ trust-dns-proto.workspace = true trust-dns-resolver.workspace = true trust-dns-server.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/dns-service-client/Cargo.toml b/dns-service-client/Cargo.toml index e351d90da2..681c06672f 100644 --- a/dns-service-client/Cargo.toml +++ b/dns-service-client/Cargo.toml @@ -14,4 +14,4 @@ serde.workspace = true serde_json.workspace = true slog.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/dpd-client/Cargo.toml b/dpd-client/Cargo.toml index 26807f7d79..0239c6d9b0 100644 --- a/dpd-client/Cargo.toml +++ b/dpd-client/Cargo.toml @@ -17,7 +17,7 @@ ipnetwork.workspace = true http.workspace = true schemars.workspace = true rand.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [build-dependencies] anyhow.workspace = true diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml index 5ff0f9b377..732a4a2091 100644 --- a/end-to-end-tests/Cargo.toml +++ b/end-to-end-tests/Cargo.toml @@ -24,4 +24,4 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } toml.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/gateway-cli/Cargo.toml b/gateway-cli/Cargo.toml index 0d179750ea..ba66fa4c4f 100644 --- a/gateway-cli/Cargo.toml +++ b/gateway-cli/Cargo.toml @@ -24,4 +24,4 @@ uuid.workspace = true gateway-client.workspace = true gateway-messages.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/gateway-client/Cargo.toml b/gateway-client/Cargo.toml index 96a1eb221f..fc33174107 100644 --- a/gateway-client/Cargo.toml +++ b/gateway-client/Cargo.toml @@ -15,4 +15,4 @@ serde_json.workspace = true schemars.workspace = true slog.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/gateway-test-utils/Cargo.toml b/gateway-test-utils/Cargo.toml index 9d80e63f05..81b7686eb2 100644 --- a/gateway-test-utils/Cargo.toml +++ b/gateway-test-utils/Cargo.toml @@ -14,4 +14,4 @@ slog.workspace = true sp-sim.workspace = true tokio.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index f5abce88e9..07934a6ad3 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -34,7 +34,7 @@ tokio-tungstenite.workspace = true tokio-util.workspace = true toml.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/illumos-utils/Cargo.toml b/illumos-utils/Cargo.toml index e292097bc5..e521b54d02 100644 --- a/illumos-utils/Cargo.toml +++ b/illumos-utils/Cargo.toml @@ -29,7 +29,7 @@ zone.workspace = true # only enabled via the `testing` feature mockall = { workspace = true, optional = true } -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [target.'cfg(target_os = "illumos")'.dependencies] opte-ioctl.workspace = true diff --git a/installinator-artifact-client/Cargo.toml b/installinator-artifact-client/Cargo.toml index 18447b8e83..c3ddc529d9 100644 --- a/installinator-artifact-client/Cargo.toml +++ b/installinator-artifact-client/Cargo.toml @@ -15,4 +15,4 @@ serde_json.workspace = true slog.workspace = true update-engine.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/installinator-artifactd/Cargo.toml b/installinator-artifactd/Cargo.toml index 9318b725db..b14ca4002f 100644 --- a/installinator-artifactd/Cargo.toml +++ b/installinator-artifactd/Cargo.toml @@ -20,7 +20,7 @@ uuid.workspace = true installinator-common.workspace = true omicron-common.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/installinator-common/Cargo.toml b/installinator-common/Cargo.toml index 0f1bf86901..8fea234e20 100644 --- a/installinator-common/Cargo.toml +++ b/installinator-common/Cargo.toml @@ -15,4 +15,4 @@ serde_json.workspace = true serde_with.workspace = true thiserror.workspace = true update-engine.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml index 428ea0d08e..a4f170ddba 100644 --- a/installinator/Cargo.toml +++ b/installinator/Cargo.toml @@ -42,7 +42,7 @@ toml.workspace = true tufaceous-lib.workspace = true update-engine.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] omicron-test-utils.workspace = true @@ -57,4 +57,4 @@ tokio-stream.workspace = true [features] image-standard = [] image-trampoline = [] -rack-topology-single-sled = [] \ No newline at end of file +rack-topology-single-sled = [] diff --git a/internal-dns-cli/Cargo.toml b/internal-dns-cli/Cargo.toml index fb5780d22a..dab92c6d7c 100644 --- a/internal-dns-cli/Cargo.toml +++ b/internal-dns-cli/Cargo.toml @@ -13,4 +13,4 @@ omicron-common.workspace = true slog.workspace = true tokio.workspace = true trust-dns-resolver.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/internal-dns/Cargo.toml b/internal-dns/Cargo.toml index d680ab3ce1..ecb2d48bda 100644 --- a/internal-dns/Cargo.toml +++ b/internal-dns/Cargo.toml @@ -17,7 +17,7 @@ thiserror.workspace = true trust-dns-proto.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] assert_matches.workspace = true diff --git a/ipcc-key-value/Cargo.toml b/ipcc-key-value/Cargo.toml index 128fde9a01..04aea9f939 100644 --- a/ipcc-key-value/Cargo.toml +++ b/ipcc-key-value/Cargo.toml @@ -11,7 +11,7 @@ omicron-common.workspace = true serde.workspace = true thiserror.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] omicron-common = { workspace = true, features = ["testing"] } diff --git a/key-manager/Cargo.toml b/key-manager/Cargo.toml index 69ae3b25bd..c44ec61ea4 100644 --- a/key-manager/Cargo.toml +++ b/key-manager/Cargo.toml @@ -14,5 +14,5 @@ slog.workspace = true thiserror.workspace = true tokio.workspace = true zeroize.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/nexus-client/Cargo.toml b/nexus-client/Cargo.toml index d59c013992..2734142f9f 100644 --- a/nexus-client/Cargo.toml +++ b/nexus-client/Cargo.toml @@ -18,4 +18,4 @@ serde.workspace = true serde_json.workspace = true slog.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 91872e2c32..3de6dac7c0 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -90,7 +90,7 @@ oximeter.workspace = true oximeter-instruments = { workspace = true, features = ["http-instruments"] } oximeter-producer.workspace = true rustls = { workspace = true } -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] async-bb8-diesel.workspace = true diff --git a/nexus/authz-macros/Cargo.toml b/nexus/authz-macros/Cargo.toml index 3d55afa477..15f18cb9c8 100644 --- a/nexus/authz-macros/Cargo.toml +++ b/nexus/authz-macros/Cargo.toml @@ -14,4 +14,4 @@ quote.workspace = true serde.workspace = true serde_tokenstream.workspace = true syn.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/nexus/db-macros/Cargo.toml b/nexus/db-macros/Cargo.toml index ce206bb56e..053c381ac9 100644 --- a/nexus/db-macros/Cargo.toml +++ b/nexus/db-macros/Cargo.toml @@ -15,7 +15,7 @@ quote.workspace = true serde.workspace = true serde_tokenstream.workspace = true syn = { workspace = true, features = ["extra-traits"] } -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] rustfmt-wrapper.workspace = true diff --git a/nexus/db-model/Cargo.toml b/nexus/db-model/Cargo.toml index aedbb9168b..a5cb9a06be 100644 --- a/nexus/db-model/Cargo.toml +++ b/nexus/db-model/Cargo.toml @@ -36,7 +36,7 @@ nexus-defaults.workspace = true nexus-types.workspace = true omicron-passwords.workspace = true sled-agent-client.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index af01c1732b..eaf3dc1295 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -63,7 +63,7 @@ nexus-types.workspace = true omicron-common.workspace = true omicron-passwords.workspace = true oximeter.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] assert_matches.workspace = true diff --git a/nexus/defaults/Cargo.toml b/nexus/defaults/Cargo.toml index 09a95fa839..0724b5bf4d 100644 --- a/nexus/defaults/Cargo.toml +++ b/nexus/defaults/Cargo.toml @@ -11,4 +11,4 @@ rand.workspace = true serde_json.workspace = true omicron-common.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/nexus/test-interface/Cargo.toml b/nexus/test-interface/Cargo.toml index e0743e84bc..0071ffaa28 100644 --- a/nexus/test-interface/Cargo.toml +++ b/nexus/test-interface/Cargo.toml @@ -12,4 +12,4 @@ nexus-types.workspace = true omicron-common.workspace = true slog.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/nexus/test-utils-macros/Cargo.toml b/nexus/test-utils-macros/Cargo.toml index 1bfa25017a..d3d28a7640 100644 --- a/nexus/test-utils-macros/Cargo.toml +++ b/nexus/test-utils-macros/Cargo.toml @@ -11,4 +11,4 @@ proc-macro = true proc-macro2.workspace = true quote.workspace = true syn = { workspace = true, features = [ "fold", "parsing" ] } -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index a2e7600e93..8eb8df4a5b 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -38,4 +38,4 @@ tempfile.workspace = true trust-dns-proto.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index f7ffafec52..c499714c31 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -25,4 +25,4 @@ api_identity.workspace = true dns-service-client.workspace = true omicron-common.workspace = true omicron-passwords.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/oxide-client/Cargo.toml b/oxide-client/Cargo.toml index df34ab9721..3cb411729d 100644 --- a/oxide-client/Cargo.toml +++ b/oxide-client/Cargo.toml @@ -21,4 +21,4 @@ thiserror.workspace = true tokio = { workspace = true, features = [ "net" ] } trust-dns-resolver.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/oximeter-client/Cargo.toml b/oximeter-client/Cargo.toml index 297dfb6c92..a8aa7de02c 100644 --- a/oximeter-client/Cargo.toml +++ b/oximeter-client/Cargo.toml @@ -12,4 +12,4 @@ reqwest = { workspace = true, features = ["json", "rustls-tls", "stream"] } serde.workspace = true slog.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/oximeter/collector/Cargo.toml b/oximeter/collector/Cargo.toml index c8c4030dba..bc8cc19634 100644 --- a/oximeter/collector/Cargo.toml +++ b/oximeter/collector/Cargo.toml @@ -22,7 +22,7 @@ thiserror.workspace = true tokio.workspace = true toml.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/oximeter/db/Cargo.toml b/oximeter/db/Cargo.toml index 77bce09db9..ad6d584b1b 100644 --- a/oximeter/db/Cargo.toml +++ b/oximeter/db/Cargo.toml @@ -25,7 +25,7 @@ thiserror.workspace = true tokio = { workspace = true, features = [ "rt-multi-thread", "macros" ] } usdt.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] itertools.workspace = true diff --git a/oximeter/instruments/Cargo.toml b/oximeter/instruments/Cargo.toml index 4adff0463a..3653ab8011 100644 --- a/oximeter/instruments/Cargo.toml +++ b/oximeter/instruments/Cargo.toml @@ -12,7 +12,7 @@ oximeter.workspace = true tokio.workspace = true http = { workspace = true, optional = true } uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true [features] default = ["http-instruments"] diff --git a/oximeter/oximeter-macro-impl/Cargo.toml b/oximeter/oximeter-macro-impl/Cargo.toml index ff116e1c9d..df9ed547ed 100644 --- a/oximeter/oximeter-macro-impl/Cargo.toml +++ b/oximeter/oximeter-macro-impl/Cargo.toml @@ -12,4 +12,4 @@ proc-macro = true proc-macro2.workspace = true quote.workspace = true syn = { workspace = true, features = [ "full", "extra-traits" ] } -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/oximeter/oximeter/Cargo.toml b/oximeter/oximeter/Cargo.toml index b2aa15f85e..7d01b8f8be 100644 --- a/oximeter/oximeter/Cargo.toml +++ b/oximeter/oximeter/Cargo.toml @@ -15,7 +15,7 @@ schemars = { workspace = true, features = [ "uuid1", "bytes", "chrono" ] } serde.workspace = true thiserror.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] approx.workspace = true diff --git a/oximeter/producer/Cargo.toml b/oximeter/producer/Cargo.toml index f171f57e8a..3f74ba753f 100644 --- a/oximeter/producer/Cargo.toml +++ b/oximeter/producer/Cargo.toml @@ -19,4 +19,4 @@ slog-dtrace.workspace = true tokio.workspace = true thiserror.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/package/Cargo.toml b/package/Cargo.toml index 9fc4610020..b840938db0 100644 --- a/package/Cargo.toml +++ b/package/Cargo.toml @@ -34,7 +34,7 @@ tokio = { workspace = true, features = [ "full" ] } toml.workspace = true topological-sort.workspace = true walkdir.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/passwords/Cargo.toml b/passwords/Cargo.toml index cbd569ef4c..8adcf75a2e 100644 --- a/passwords/Cargo.toml +++ b/passwords/Cargo.toml @@ -11,7 +11,7 @@ thiserror.workspace = true schemars.workspace = true serde.workspace = true serde_with.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] argon2alt = { package = "rust-argon2", version = "1.0" } diff --git a/rpaths/Cargo.toml b/rpaths/Cargo.toml index 7671be4968..45e6c9b925 100644 --- a/rpaths/Cargo.toml +++ b/rpaths/Cargo.toml @@ -5,4 +5,4 @@ edition = "2021" license = "MPL-2.0" [dependencies] -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/sled-agent-client/Cargo.toml b/sled-agent-client/Cargo.toml index 01c1032a51..b2ed07caba 100644 --- a/sled-agent-client/Cargo.toml +++ b/sled-agent-client/Cargo.toml @@ -15,4 +15,4 @@ reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } serde.workspace = true slog.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index d4ccfc97c8..82d7411d1a 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -76,7 +76,7 @@ uuid.workspace = true zeroize.workspace = true zone.workspace = true static_assertions.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [target.'cfg(target_os = "illumos")'.dependencies] opte-ioctl.workspace = true diff --git a/sled-hardware/Cargo.toml b/sled-hardware/Cargo.toml index 880f93441c..14ae15996b 100644 --- a/sled-hardware/Cargo.toml +++ b/sled-hardware/Cargo.toml @@ -24,7 +24,7 @@ thiserror.workspace = true tofino.workspace = true tokio.workspace = true uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [target.'cfg(target_os = "illumos")'.dependencies] illumos-devinfo = { git = "https://github.com/oxidecomputer/illumos-devinfo", branch = "main" } diff --git a/sp-sim/Cargo.toml b/sp-sim/Cargo.toml index 2a1ae19468..07d956e41e 100644 --- a/sp-sim/Cargo.toml +++ b/sp-sim/Cargo.toml @@ -21,7 +21,7 @@ sprockets-rot.workspace = true thiserror.workspace = true tokio = { workspace = true, features = [ "full" ] } toml.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [[bin]] name = "sp-sim" diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index a0227a4de2..9e21f3ca12 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -25,7 +25,7 @@ usdt.workspace = true rcgen.workspace = true regex.workspace = true reqwest.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/tufaceous-lib/Cargo.toml b/tufaceous-lib/Cargo.toml index 8b5c4fa7ca..bcfcee6b9c 100644 --- a/tufaceous-lib/Cargo.toml +++ b/tufaceous-lib/Cargo.toml @@ -32,7 +32,7 @@ toml.workspace = true tough.workspace = true url = "2.4.1" zip.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] omicron-test-utils.workspace = true diff --git a/tufaceous/Cargo.toml b/tufaceous/Cargo.toml index f3e3b815d2..e48513e24c 100644 --- a/tufaceous/Cargo.toml +++ b/tufaceous/Cargo.toml @@ -18,7 +18,7 @@ slog-async.workspace = true slog-envlogger.workspace = true slog-term.workspace = true tufaceous-lib.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] assert_cmd.workspace = true diff --git a/update-engine/Cargo.toml b/update-engine/Cargo.toml index 25ade83f34..af988bf091 100644 --- a/update-engine/Cargo.toml +++ b/update-engine/Cargo.toml @@ -21,7 +21,7 @@ schemars = { workspace = true, features = ["uuid1"] } slog.workspace = true tokio = { workspace = true, features = ["macros", "sync", "time", "rt-multi-thread"] } uuid.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] buf-list.workspace = true diff --git a/wicket-common/Cargo.toml b/wicket-common/Cargo.toml index 229561cd38..b87e742133 100644 --- a/wicket-common/Cargo.toml +++ b/wicket-common/Cargo.toml @@ -13,4 +13,4 @@ serde.workspace = true serde_json.workspace = true thiserror.workspace = true update-engine.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/wicket-dbg/Cargo.toml b/wicket-dbg/Cargo.toml index bc22424c69..e7e8a58468 100644 --- a/wicket-dbg/Cargo.toml +++ b/wicket-dbg/Cargo.toml @@ -22,7 +22,7 @@ wicket.workspace = true # used only by wicket-dbg binary reedline = "0.23.0" -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [[bin]] name = "wicket-dbg" diff --git a/wicket/Cargo.toml b/wicket/Cargo.toml index 58605c8037..5392e72e9f 100644 --- a/wicket/Cargo.toml +++ b/wicket/Cargo.toml @@ -46,7 +46,7 @@ omicron-passwords.workspace = true update-engine.workspace = true wicket-common.workspace = true wicketd-client.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [dev-dependencies] assert_cmd.workspace = true diff --git a/wicketd-client/Cargo.toml b/wicketd-client/Cargo.toml index 2d959f1f8d..814309b975 100644 --- a/wicketd-client/Cargo.toml +++ b/wicketd-client/Cargo.toml @@ -18,4 +18,4 @@ slog.workspace = true update-engine.workspace = true uuid.workspace = true wicket-common.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index 6df5e0e4e5..1044e1ff51 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -54,7 +54,7 @@ sled-hardware.workspace = true tufaceous-lib.workspace = true update-engine.workspace = true wicket-common.workspace = true -omicron-workspace-hack = { version = "0.1", path = "../workspace-hack" } +omicron-workspace-hack.workspace = true [[bin]] name = "wicketd" From b0487d3777a87edf7cb7b3a11badf47a75931323 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Wed, 4 Oct 2023 17:08:49 -0500 Subject: [PATCH 19/89] Bump web console (#4204) The only functional change should be the form validation one. And one icon. But there are some tweaks to our build setup that I'd like to make sure we didn't mess up. https://github.com/oxidecomputer/console/compare/af6536d5...0cc1e03a * [0cc1e03a](https://github.com/oxidecomputer/console/commit/0cc1e03a) oxidecomputer/console#1770 * [48aea2f4](https://github.com/oxidecomputer/console/commit/48aea2f4) npm audit fix * [84aff1de](https://github.com/oxidecomputer/console/commit/84aff1de) oxidecomputer/console#1769 * [c127febd](https://github.com/oxidecomputer/console/commit/c127febd) oxidecomputer/console#1768 * [8c9513c1](https://github.com/oxidecomputer/console/commit/8c9513c1) oxidecomputer/console#1765 * [0314fd72](https://github.com/oxidecomputer/console/commit/0314fd72) oxidecomputer/console#1742 * [8918ffa9](https://github.com/oxidecomputer/console/commit/8918ffa9) skip the other flaky test in safari for now. I'm suffering * [b357246e](https://github.com/oxidecomputer/console/commit/b357246e) increase playwright total time to 20 minutes * [4f7d401d](https://github.com/oxidecomputer/console/commit/4f7d401d) be sneakier about PR numbers in commit messages in bump omicron PR --- tools/console_version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/console_version b/tools/console_version index dba32c3e94..0c30c707e1 100644 --- a/tools/console_version +++ b/tools/console_version @@ -1,2 +1,2 @@ -COMMIT="af6536d587a17a65398407ca03d364345aa24342" -SHA2="00701652eb1e495fd22409dcdf74ebae2ba081529f65fb41c5ac3a2fef50a149" +COMMIT="0cc1e03a24b3f5da275d15b969978a385d6b3b27" +SHA2="46a186fc3bf919a3aa2871aeab8441e4a13ed134f912b5d76c7ff891fed66cee" From ce81dd12e5a69c50979f8c5049a17d84dbdc0a01 Mon Sep 17 00:00:00 2001 From: bnaecker Date: Wed, 4 Oct 2023 15:39:43 -0700 Subject: [PATCH 20/89] Adds functionality to run oximeter standalone (#4117) - Adds a "standalone" mode for the `oximeter-collector` crate, including the binary and main inner types. This runs in a slightly different mode, in which the ClickHouse database itself isn't strictly required. In this case, a task to simply print the results will be spawned in place of the normal results-sink task which inserts records into the database. - Creates a tiny fake Nexus server, which includes only the API needed to register collectors and producers. This is started automatically when running `oximeter standalone`, and used to assign producers / collectors as the real Nexus does, but without a database. The assignments are only in memory. - Adds internal `oximeter` API for listing / deleting a producer for each oximeter collector, and an `omdb` subcommand which exercises the listing. --- Cargo.lock | 12 + common/src/api/internal/nexus.rs | 2 +- dev-tools/omdb/Cargo.toml | 2 + dev-tools/omdb/src/bin/omdb/main.rs | 4 + dev-tools/omdb/src/bin/omdb/oximeter.rs | 94 ++++ dev-tools/omdb/tests/usage_errors.out | 2 + docs/how-to-run.adoc | 34 ++ openapi/oximeter.json | 130 +++++ oximeter-client/Cargo.toml | 1 + oximeter/collector/Cargo.toml | 7 + oximeter/collector/src/bin/oximeter.rs | 103 +++- oximeter/collector/src/lib.rs | 473 ++++++++++++++++-- oximeter/collector/src/standalone.rs | 263 ++++++++++ .../tests/output/cmd-oximeter-noargs-stderr | 8 +- oximeter/producer/Cargo.toml | 4 + oximeter/producer/examples/producer.rs | 45 +- oximeter/producer/src/lib.rs | 142 ++++-- 17 files changed, 1215 insertions(+), 111 deletions(-) create mode 100644 dev-tools/omdb/src/bin/omdb/oximeter.rs create mode 100644 oximeter/collector/src/standalone.rs diff --git a/Cargo.lock b/Cargo.lock index c6335cb32e..b931918b9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5125,6 +5125,7 @@ dependencies = [ "diesel", "dropshot", "expectorate", + "futures", "humantime", "internal-dns 0.1.0", "ipnetwork", @@ -5139,6 +5140,7 @@ dependencies = [ "omicron-rpaths", "omicron-test-utils", "omicron-workspace-hack", + "oximeter-client", "pq-sys", "regex", "serde", @@ -5716,6 +5718,7 @@ name = "oximeter-client" version = "0.1.0" dependencies = [ "chrono", + "futures", "omicron-common 0.1.0", "omicron-workspace-hack", "progenitor", @@ -5729,24 +5732,31 @@ dependencies = [ name = "oximeter-collector" version = "0.1.0" dependencies = [ + "anyhow", "clap 4.4.3", "dropshot", "expectorate", "futures", "internal-dns 0.1.0", "nexus-client 0.1.0", + "nexus-types", "omicron-common 0.1.0", "omicron-test-utils", "omicron-workspace-hack", "openapi-lint", "openapiv3", "oximeter 0.1.0", + "oximeter-client", "oximeter-db", + "rand 0.8.5", "reqwest", + "schemars", "serde", "serde_json", "slog", + "slog-async", "slog-dtrace", + "slog-term", "subprocess", "thiserror", "tokio", @@ -5821,7 +5831,9 @@ dependencies = [ name = "oximeter-producer" version = "0.1.0" dependencies = [ + "anyhow", "chrono", + "clap 4.4.3", "dropshot", "nexus-client 0.1.0", "omicron-common 0.1.0", diff --git a/common/src/api/internal/nexus.rs b/common/src/api/internal/nexus.rs index 018869ce14..983976bbb7 100644 --- a/common/src/api/internal/nexus.rs +++ b/common/src/api/internal/nexus.rs @@ -67,7 +67,7 @@ pub struct InstanceRuntimeState { /// Information announced by a metric server, used so that clients can contact it and collect /// available metric data from it. -#[derive(Debug, Clone, JsonSchema, Serialize, Deserialize)] +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] pub struct ProducerEndpoint { pub id: Uuid, pub address: SocketAddr, diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml index f865acff2b..cd4af6e947 100644 --- a/dev-tools/omdb/Cargo.toml +++ b/dev-tools/omdb/Cargo.toml @@ -16,11 +16,13 @@ diesel.workspace = true dropshot.workspace = true humantime.workspace = true internal-dns.workspace = true +futures.workspace = true nexus-client.workspace = true nexus-db-model.workspace = true nexus-db-queries.workspace = true nexus-types.workspace = true omicron-common.workspace = true +oximeter-client.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. pq-sys = "*" serde.workspace = true diff --git a/dev-tools/omdb/src/bin/omdb/main.rs b/dev-tools/omdb/src/bin/omdb/main.rs index 166ed3043f..d1a56e1d80 100644 --- a/dev-tools/omdb/src/bin/omdb/main.rs +++ b/dev-tools/omdb/src/bin/omdb/main.rs @@ -42,6 +42,7 @@ use std::net::SocketAddrV6; mod db; mod nexus; +mod oximeter; mod sled_agent; #[tokio::main] @@ -57,6 +58,7 @@ async fn main() -> Result<(), anyhow::Error> { match &args.command { OmdbCommands::Db(db) => db.run_cmd(&args, &log).await, OmdbCommands::Nexus(nexus) => nexus.run_cmd(&args, &log).await, + OmdbCommands::Oximeter(oximeter) => oximeter.run_cmd(&log).await, OmdbCommands::SledAgent(sled) => sled.run_cmd(&args, &log).await, } } @@ -155,6 +157,8 @@ enum OmdbCommands { Db(db::DbArgs), /// Debug a specific Nexus instance Nexus(nexus::NexusArgs), + /// Query oximeter collector state + Oximeter(oximeter::OximeterArgs), /// Debug a specific Sled SledAgent(sled_agent::SledAgentArgs), } diff --git a/dev-tools/omdb/src/bin/omdb/oximeter.rs b/dev-tools/omdb/src/bin/omdb/oximeter.rs new file mode 100644 index 0000000000..e0f20556a2 --- /dev/null +++ b/dev-tools/omdb/src/bin/omdb/oximeter.rs @@ -0,0 +1,94 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! omdb commands that query oximeter + +use anyhow::Context; +use clap::Args; +use clap::Subcommand; +use futures::TryStreamExt; +use oximeter_client::types::ProducerEndpoint; +use oximeter_client::Client; +use slog::Logger; +use std::net::SocketAddr; +use std::time::Duration; +use tabled::Table; +use tabled::Tabled; +use uuid::Uuid; + +#[derive(Debug, Args)] +pub struct OximeterArgs { + /// URL of the oximeter collector to query + #[arg(long, env("OMDB_OXIMETER_URL"))] + oximeter_url: String, + + #[command(subcommand)] + command: OximeterCommands, +} + +/// Subcommands that query oximeter collector state +#[derive(Debug, Subcommand)] +enum OximeterCommands { + /// List the producers the collector is assigned to poll + ListProducers, +} + +impl OximeterArgs { + fn client(&self, log: &Logger) -> Client { + Client::new( + &self.oximeter_url, + log.new(slog::o!("component" => "oximeter-client")), + ) + } + + pub async fn run_cmd(&self, log: &Logger) -> anyhow::Result<()> { + let client = self.client(log); + match self.command { + OximeterCommands::ListProducers => { + self.list_producers(client).await + } + } + } + + async fn list_producers(&self, client: Client) -> anyhow::Result<()> { + let info = client + .collector_info() + .await + .context("failed to fetch collector info")?; + let producers: Vec = client + .producers_list_stream(None) + .map_ok(Producer::from) + .try_collect() + .await + .context("failed to list producers")?; + let table = Table::new(producers) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("Collector ID: {}\n", info.id); + println!("{table}"); + Ok(()) + } +} + +#[derive(Tabled)] +#[tabled(rename_all = "SCREAMING_SNAKE_CASE")] +struct Producer { + id: Uuid, + address: SocketAddr, + base_route: String, + interval: String, +} + +impl From for Producer { + fn from(p: ProducerEndpoint) -> Self { + let interval = Duration::new(p.interval.secs, p.interval.nanos); + Self { + id: p.id, + address: p.address.parse().unwrap(), + base_route: p.base_route, + interval: humantime::format_duration(interval).to_string(), + } + } +} diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index b5421b76af..dc2a16bc47 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -11,6 +11,7 @@ Usage: omdb [OPTIONS] Commands: db Query the control plane database (CockroachDB) nexus Debug a specific Nexus instance + oximeter Query oximeter collector state sled-agent Debug a specific Sled help Print this message or the help of the given subcommand(s) @@ -33,6 +34,7 @@ Usage: omdb [OPTIONS] Commands: db Query the control plane database (CockroachDB) nexus Debug a specific Nexus instance + oximeter Query oximeter collector state sled-agent Debug a specific Sled help Print this message or the help of the given subcommand(s) diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 7539c5183f..aa1ee3c73d 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -697,3 +697,37 @@ To build a recovery host image: ---- $ ./tools/build-host-image.sh -R $HELIOS_PATH /work/trampoline-global-zone-packages.tar.gz ---- + + +== Running `oximeter` in standalone mode + +`oximeter` is the program used to collect metrics from producers in the control +plane. Normally, the producers register themselves with `nexus`, which creates a +durable assignment between the producer and an `oximeter` collector in the +database. That allows components to survive restarts, while still producing +metrics. + +To ease development, `oximeter` can be run in "standalone" mode. In this case, a +mock `nexus` server is started, with only the minimal subset of the internal API +needed to register producers and collectors. Neither CockroachDB nor ClickHouse +is required, although ClickHouse _can_ be used, if one wants to see how data is +inserted into the database. + +To run `oximeter` in standalone, use: + +[source,console] +---- +$ cargo run --bin oximeter -- standalone +---- + +The producer should still register with `nexus` as normal, which is usually done +with an explicit IP address and port. This defaults to `[::1]:12221`. + +When run this way, `oximeter` will print the samples it collects from the +producers to its logs, like so: + +[source,console] +---- +Sep 26 17:48:56.006 INFO sample: Sample { measurement: Measurement { timestamp: 2023-09-26T17:48:56.004565890Z, datum: CumulativeF64(Cumulative { start_time: 2023-09-26T17:48:45.997404777Z, value: 10.007154703 }) }, timeseries_name: "virtual_machine:cpu_busy", target: FieldSet { name: "virtual_machine", fields: {"instance_id": Field { name: "instance_id", value: Uuid(564ef6df-d5f6-4204-88f7-5c615859cfa7) }, "project_id": Field { name: "project_id", value: Uuid(2dc7e1c9-f8ac-49d7-8292-46e9e2b1a61d) }} }, metric: FieldSet { name: "cpu_busy", fields: {"cpu_id": Field { name: "cpu_id", value: I64(0) }} } }, component: results-sink, collector_id: 78c7c9a5-1569-460a-8899-aada9ad5db6c, component: oximeter-standalone, component: nexus-standalone, file: oximeter/collector/src/lib.rs:280 +Sep 26 17:48:56.006 INFO sample: Sample { measurement: Measurement { timestamp: 2023-09-26T17:48:56.004700841Z, datum: CumulativeF64(Cumulative { start_time: 2023-09-26T17:48:45.997405187Z, value: 10.007154703 }) }, timeseries_name: "virtual_machine:cpu_busy", target: FieldSet { name: "virtual_machine", fields: {"instance_id": Field { name: "instance_id", value: Uuid(564ef6df-d5f6-4204-88f7-5c615859cfa7) }, "project_id": Field { name: "project_id", value: Uuid(2dc7e1c9-f8ac-49d7-8292-46e9e2b1a61d) }} }, metric: FieldSet { name: "cpu_busy", fields: {"cpu_id": Field { name: "cpu_id", value: I64(1) }} } }, component: results-sink, collector_id: 78c7c9a5-1569-460a-8899-aada9ad5db6c, component: oximeter-standalone, component: nexus-standalone, file: oximeter/collector/src/lib.rs:280 +---- diff --git a/openapi/oximeter.json b/openapi/oximeter.json index 6781b77892..ebc7957c2e 100644 --- a/openapi/oximeter.json +++ b/openapi/oximeter.json @@ -10,7 +10,76 @@ "version": "0.0.1" }, "paths": { + "/info": { + "get": { + "operationId": "collector_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CollectorInfo" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/producers": { + "get": { + "operationId": "producers_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProducerEndpointResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + }, "post": { "operationId": "producers_post", "requestBody": { @@ -35,6 +104,33 @@ } } } + }, + "/producers/{producer_id}": { + "delete": { + "operationId": "producer_delete", + "parameters": [ + { + "in": "path", + "name": "producer_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } } }, "components": { @@ -51,6 +147,19 @@ } }, "schemas": { + "CollectorInfo": { + "type": "object", + "properties": { + "id": { + "description": "The collector's UUID.", + "type": "string", + "format": "uuid" + } + }, + "required": [ + "id" + ] + }, "Duration": { "type": "object", "properties": { @@ -113,6 +222,27 @@ "id", "interval" ] + }, + "ProducerEndpointResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/ProducerEndpoint" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] } } } diff --git a/oximeter-client/Cargo.toml b/oximeter-client/Cargo.toml index a8aa7de02c..e54b152415 100644 --- a/oximeter-client/Cargo.toml +++ b/oximeter-client/Cargo.toml @@ -6,6 +6,7 @@ license = "MPL-2.0" [dependencies] chrono.workspace = true +futures.workspace = true omicron-common.workspace = true progenitor.workspace = true reqwest = { workspace = true, features = ["json", "rustls-tls", "stream"] } diff --git a/oximeter/collector/Cargo.toml b/oximeter/collector/Cargo.toml index bc8cc19634..470d9db312 100644 --- a/oximeter/collector/Cargo.toml +++ b/oximeter/collector/Cargo.toml @@ -6,18 +6,25 @@ description = "The oximeter metric collection server" license = "MPL-2.0" [dependencies] +anyhow.workspace = true clap.workspace = true dropshot.workspace = true futures.workspace = true internal-dns.workspace = true nexus-client.workspace = true +nexus-types.workspace = true omicron-common.workspace = true oximeter.workspace = true +oximeter-client.workspace = true oximeter-db.workspace = true +rand.workspace = true reqwest = { workspace = true, features = [ "json" ] } +schemars.workspace = true serde.workspace = true slog.workspace = true +slog-async.workspace = true slog-dtrace.workspace = true +slog-term.workspace = true thiserror.workspace = true tokio.workspace = true toml.workspace = true diff --git a/oximeter/collector/src/bin/oximeter.rs b/oximeter/collector/src/bin/oximeter.rs index bf54cf33fa..8c4bf0e27c 100644 --- a/oximeter/collector/src/bin/oximeter.rs +++ b/oximeter/collector/src/bin/oximeter.rs @@ -3,12 +3,21 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. //! Main entry point to run an `oximeter` server in the control plane. -// Copyright 2021 Oxide Computer Company + +// Copyright 2023 Oxide Computer Company use clap::Parser; use omicron_common::cmd::fatal; use omicron_common::cmd::CmdError; -use oximeter_collector::{oximeter_api, Config, Oximeter, OximeterArguments}; +use oximeter_collector::oximeter_api; +use oximeter_collector::standalone_nexus_api; +use oximeter_collector::Config; +use oximeter_collector::Oximeter; +use oximeter_collector::OximeterArguments; +use oximeter_collector::StandaloneNexus; +use slog::Level; +use std::net::Ipv6Addr; +use std::net::SocketAddr; use std::net::SocketAddrV6; use std::path::PathBuf; use uuid::Uuid; @@ -23,6 +32,16 @@ pub fn run_openapi() -> Result<(), String> { .map_err(|e| e.to_string()) } +pub fn run_standalone_openapi() -> Result<(), String> { + standalone_nexus_api() + .openapi("Oxide Nexus API", "0.0.1") + .description("API for interacting with Nexus") + .contact_url("https://oxide.computer") + .contact_email("api@oxide.computer") + .write(&mut std::io::stdout()) + .map_err(|e| e.to_string()) +} + /// Run an oximeter metric collection server in the Oxide Control Plane. #[derive(Parser)] #[clap(name = "oximeter", about = "See README.adoc for more information")] @@ -36,12 +55,71 @@ enum Args { #[clap(name = "CONFIG_FILE", action)] config_file: PathBuf, + /// The UUID for this instance of the `oximeter` collector. #[clap(short, long, action)] id: Uuid, + /// The socket address at which `oximeter`'s HTTP server runs. #[clap(short, long, action)] address: SocketAddrV6, }, + + /// Run `oximeter` in standalone mode for development. + /// + /// In this mode, `oximeter` can be used to test the collection of metrics + /// from producers, without requiring all the normal machinery of the + /// control plane. The collector is run as usual, but additionally starts a + /// API server to stand-in for Nexus. The registrations of the producers and + /// collectors occurs through the normal code path, but uses this mock Nexus + /// instead of the real thing. + Standalone { + /// The ID for the collector. + /// + /// Default is to generate a new, random UUID. + #[arg(long, default_value_t = Uuid::new_v4())] + id: Uuid, + + /// Address at which `oximeter` itself listens. + /// + /// This address can be used to register new producers, after the + /// program has already started. + #[arg( + long, + default_value_t = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 12223, 0, 0) + )] + address: SocketAddrV6, + + /// The address for the mock Nexus server used to register. + /// + /// This program starts a mock version of Nexus, which is used only to + /// register the producers and collectors. This allows them to operate + /// as they usually would, registering each other with Nexus so that an + /// assignment between them can be made. + #[arg( + long, + default_value_t = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 12221, 0, 0) + )] + nexus: SocketAddrV6, + + /// The address for ClickHouse. + /// + /// If not provided, `oximeter` will not attempt to insert records into + /// the database at all. In this mode, the program will print the + /// collected samples, instead of inserting them into the database. + #[arg(long)] + clickhouse: Option, + + /// The log-level. + #[arg(long, default_value_t = Level::Info, value_parser = parse_log_level)] + log_level: Level, + }, + + /// Print the fake Nexus's standalone API. + StandaloneOpenapi, +} + +fn parse_log_level(s: &str) -> Result { + s.parse().map_err(|_| "Invalid log level".to_string()) } #[tokio::main] @@ -65,5 +143,26 @@ async fn do_run() -> Result<(), CmdError> { .await .map_err(|e| CmdError::Failure(e.to_string())) } + Args::Standalone { id, address, nexus, clickhouse, log_level } => { + // Start the standalone Nexus server, for registration of both the + // collector and producers. + let nexus_server = StandaloneNexus::new(nexus.into(), log_level) + .map_err(|e| CmdError::Failure(e.to_string()))?; + let args = OximeterArguments { id, address }; + Oximeter::new_standalone( + nexus_server.log(), + &args, + nexus_server.local_addr(), + clickhouse, + ) + .await + .unwrap() + .serve_forever() + .await + .map_err(|e| CmdError::Failure(e.to_string())) + } + Args::StandaloneOpenapi => { + run_standalone_openapi().map_err(CmdError::Failure) + } } } diff --git a/oximeter/collector/src/lib.rs b/oximeter/collector/src/lib.rs index bf75b567ea..6674d65ecd 100644 --- a/oximeter/collector/src/lib.rs +++ b/oximeter/collector/src/lib.rs @@ -4,35 +4,71 @@ //! Implementation of the `oximeter` metric collection server. -// Copyright 2021 Oxide Computer Company - -use dropshot::{ - endpoint, ApiDescription, ConfigDropshot, ConfigLogging, HttpError, - HttpResponseUpdatedNoContent, HttpServer, HttpServerStarter, - RequestContext, TypedBody, -}; -use internal_dns::resolver::{ResolveError, Resolver}; +// Copyright 2023 Oxide Computer Company + +use anyhow::anyhow; +use anyhow::Context; +use dropshot::endpoint; +use dropshot::ApiDescription; +use dropshot::ConfigDropshot; +use dropshot::ConfigLogging; +use dropshot::EmptyScanParams; +use dropshot::HttpError; +use dropshot::HttpResponseDeleted; +use dropshot::HttpResponseOk; +use dropshot::HttpResponseUpdatedNoContent; +use dropshot::HttpServer; +use dropshot::HttpServerStarter; +use dropshot::PaginationParams; +use dropshot::Query; +use dropshot::RequestContext; +use dropshot::ResultsPage; +use dropshot::TypedBody; +use dropshot::WhichPage; +use internal_dns::resolver::ResolveError; +use internal_dns::resolver::Resolver; use internal_dns::ServiceName; -use omicron_common::address::{CLICKHOUSE_PORT, NEXUS_INTERNAL_PORT}; +use omicron_common::address::CLICKHOUSE_PORT; +use omicron_common::address::NEXUS_INTERNAL_PORT; use omicron_common::api::internal::nexus::ProducerEndpoint; -use omicron_common::{backoff, FileKv}; -use oximeter::types::{ProducerResults, ProducerResultsItem}; -use oximeter_db::{Client, DbWrite}; -use serde::{Deserialize, Serialize}; -use slog::{debug, error, info, o, trace, warn, Drain, Logger}; -use std::collections::{btree_map::Entry, BTreeMap}; -use std::net::{SocketAddr, SocketAddrV6}; +use omicron_common::backoff; +use omicron_common::FileKv; +use oximeter::types::ProducerResults; +use oximeter::types::ProducerResultsItem; +use oximeter_db::Client; +use oximeter_db::DbWrite; +use serde::Deserialize; +use serde::Serialize; +use slog::debug; +use slog::error; +use slog::info; +use slog::o; +use slog::trace; +use slog::warn; +use slog::Drain; +use slog::Logger; +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::net::SocketAddrV6; +use std::ops::Bound; use std::path::Path; use std::sync::Arc; use std::time::Duration; use thiserror::Error; -use tokio::{ - sync::mpsc, sync::oneshot, sync::Mutex, task::JoinHandle, time::interval, -}; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; +use tokio::time::interval; use uuid::Uuid; +mod standalone; +pub use standalone::standalone_nexus_api; +pub use standalone::Server as StandaloneNexus; + /// Errors collecting metric data -#[derive(Debug, Clone, Error)] +#[derive(Debug, Error)] pub enum Error { #[error("Error running Oximeter collector server: {0}")] Server(String), @@ -45,6 +81,48 @@ pub enum Error { #[error(transparent)] ResolveError(#[from] ResolveError), + + #[error("No producer is registered with ID")] + NoSuchProducer(Uuid), + + #[error("Error running standalone")] + Standalone(#[from] anyhow::Error), +} + +impl From for HttpError { + fn from(e: Error) -> Self { + match e { + Error::NoSuchProducer(id) => HttpError::for_not_found( + None, + format!("No such producer: {id}"), + ), + _ => HttpError::for_internal_error(e.to_string()), + } + } +} + +/// A simple representation of a producer, used mostly for standalone mode. +/// +/// These are usually specified as a structured string, formatted like: +/// `"@
"`. +#[derive(Copy, Clone, Debug)] +pub struct ProducerInfo { + /// The ID of the producer. + pub id: Uuid, + /// The address on which the producer listens. + pub address: SocketAddr, +} + +impl std::str::FromStr for ProducerInfo { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + let (id, addr) = s + .split_once('@') + .context("Producer info should written as @
")?; + let id = id.parse().context("Invalid UUID")?; + let address = addr.parse().context("Invalid address")?; + Ok(Self { id, address }) + } } type CollectionToken = oneshot::Sender<()>; @@ -61,7 +139,6 @@ enum CollectionMessage { // from its producer. Update(ProducerEndpoint), // Request that the task exit - #[allow(dead_code)] Shutdown, } @@ -72,7 +149,7 @@ async fn perform_collection( outbox: &mpsc::Sender<(Option, ProducerResults)>, token: Option, ) { - info!(log, "collecting from producer"); + debug!(log, "collecting from producer"); let res = client .get(format!( "http://{}{}", @@ -187,6 +264,44 @@ struct CollectionTask { pub task: JoinHandle<()>, } +// A task run by `oximeter` in standalone mode, which simply prints results as +// they're received. +async fn results_printer( + log: Logger, + mut rx: mpsc::Receiver<(Option, ProducerResults)>, +) { + loop { + match rx.recv().await { + Some((_, results)) => { + for res in results.into_iter() { + match res { + ProducerResultsItem::Ok(samples) => { + for sample in samples.into_iter() { + info!( + log, + ""; + "sample" => ?sample, + ); + } + } + ProducerResultsItem::Err(e) => { + error!( + log, + "received error from a producer"; + "err" => ?e, + ); + } + } + } + } + None => { + debug!(log, "result queue closed, exiting"); + return; + } + } + } +} + // Aggregation point for all results, from all collection tasks. async fn results_sink( log: Logger, @@ -286,6 +401,20 @@ pub struct DbConfig { pub batch_interval: u64, } +impl DbConfig { + pub const DEFAULT_BATCH_SIZE: usize = 1000; + pub const DEFAULT_BATCH_INTERVAL: u64 = 5; + + // Construct config with an address, using the defaults for other fields + fn with_address(address: SocketAddr) -> Self { + Self { + address: Some(address), + batch_size: Self::DEFAULT_BATCH_SIZE, + batch_interval: Self::DEFAULT_BATCH_INTERVAL, + } + } +} + /// The internal agent the oximeter server uses to collect metrics from producers. #[derive(Debug)] pub struct OximeterAgent { @@ -295,7 +424,8 @@ pub struct OximeterAgent { // Handle to the TX-side of a channel for collecting results from the collection tasks result_sender: mpsc::Sender<(Option, ProducerResults)>, // The actual tokio tasks running the collection on a timer. - collection_tasks: Arc>>, + collection_tasks: + Arc>>, } impl OximeterAgent { @@ -307,7 +437,10 @@ impl OximeterAgent { log: &Logger, ) -> Result { let (result_sender, result_receiver) = mpsc::channel(8); - let log = log.new(o!("component" => "oximeter-agent", "collector_id" => id.to_string())); + let log = log.new(o!( + "component" => "oximeter-agent", + "collector_id" => id.to_string(), + )); let insertion_log = log.new(o!("component" => "results-sink")); // Construct the ClickHouse client first, propagate an error if we can't reach the @@ -347,6 +480,61 @@ impl OximeterAgent { }) } + /// Construct a new standalone `oximeter` collector. + pub async fn new_standalone( + id: Uuid, + db_config: Option, + log: &Logger, + ) -> Result { + let (result_sender, result_receiver) = mpsc::channel(8); + let log = log.new(o!( + "component" => "oximeter-standalone", + "collector_id" => id.to_string(), + )); + + // If we have configuration for ClickHouse, we'll spawn the results + // sink task as usual. If not, we'll spawn a dummy task that simply + // prints the results as they're received. + let insertion_log = log.new(o!("component" => "results-sink")); + if let Some(db_config) = db_config { + let Some(address) = db_config.address else { + return Err(Error::Standalone(anyhow!( + "Must provide explicit IP address in standalone mode" + ))); + }; + let client = Client::new(address, &log); + let replicated = client.is_oximeter_cluster().await?; + if !replicated { + client.init_single_node_db().await?; + } else { + client.init_replicated_db().await?; + } + + // Spawn the task for aggregating and inserting all metrics + tokio::spawn(async move { + results_sink( + insertion_log, + client, + db_config.batch_size, + Duration::from_secs(db_config.batch_interval), + result_receiver, + ) + .await + }); + } else { + tokio::spawn(results_printer(insertion_log, result_receiver)); + } + + // Construct the ClickHouse client first, propagate an error if we can't reach the + // database. + Ok(Self { + id, + log, + result_sender, + collection_tasks: Arc::new(Mutex::new(BTreeMap::new())), + }) + } + /// Register a new producer with this oximeter instance. pub async fn register_producer( &self, @@ -355,30 +543,36 @@ impl OximeterAgent { let id = info.id; match self.collection_tasks.lock().await.entry(id) { Entry::Vacant(value) => { - info!(self.log, "registered new metric producer"; - "producer_id" => id.to_string(), - "address" => info.address, + debug!( + self.log, + "registered new metric producer"; + "producer_id" => id.to_string(), + "address" => info.address, ); // Build channel to control the task and receive results. let (tx, rx) = mpsc::channel(4); let q = self.result_sender.clone(); let log = self.log.new(o!("component" => "collection-task", "producer_id" => id.to_string())); + let info_clone = info.clone(); let task = tokio::spawn(async move { - collection_task(log, info, rx, q).await; + collection_task(log, info_clone, rx, q).await; }); - value.insert(CollectionTask { inbox: tx, task }); + value.insert((info, CollectionTask { inbox: tx, task })); } - Entry::Occupied(value) => { - info!( + Entry::Occupied(mut value) => { + debug!( self.log, - "received request to register existing metric producer, updating collection information"; + "received request to register existing metric \ + producer, updating collection information"; "producer_id" => id.to_string(), "interval" => ?info.interval, "address" => info.address, ); + value.get_mut().0 = info.clone(); value .get() + .1 .inbox .send(CollectionMessage::Update(info)) .await @@ -395,10 +589,10 @@ impl OximeterAgent { pub async fn force_collection(&self) { let mut collection_oneshots = vec![]; let collection_tasks = self.collection_tasks.lock().await; - for task in collection_tasks.iter() { + for (_id, (_endpoint, task)) in collection_tasks.iter() { let (tx, rx) = oneshot::channel(); // Scrape from each producer, into oximeter... - task.1.inbox.send(CollectionMessage::Collect(tx)).await.unwrap(); + task.inbox.send(CollectionMessage::Collect(tx)).await.unwrap(); // ... and keep track of the token that indicates once the metric // has made it into Clickhouse. collection_oneshots.push(rx); @@ -412,6 +606,55 @@ impl OximeterAgent { // successfully, or an error occurred in the collection pathway. futures::future::join_all(collection_oneshots).await; } + + /// List existing producers. + pub async fn list_producers( + &self, + start_id: Option, + limit: usize, + ) -> Vec { + let start = if let Some(id) = start_id { + Bound::Excluded(id) + } else { + Bound::Unbounded + }; + self.collection_tasks + .lock() + .await + .range((start, Bound::Unbounded)) + .take(limit) + .map(|(_id, (info, _t))| info.clone()) + .collect() + } + + /// Delete a producer by ID, stopping its collection task. + pub async fn delete_producer(&self, id: Uuid) -> Result<(), Error> { + let (_info, task) = self + .collection_tasks + .lock() + .await + .remove(&id) + .ok_or_else(|| Error::NoSuchProducer(id))?; + debug!( + self.log, + "removed collection task from set"; + "producer_id" => %id, + ); + match task.inbox.send(CollectionMessage::Shutdown).await { + Ok(_) => debug!( + self.log, + "shut down collection task"; + "producer_id" => %id, + ), + Err(e) => error!( + self.log, + "failed to shut down collection task"; + "producer_id" => %id, + "error" => ?e, + ), + } + Ok(()) + } } /// Configuration used to initialize an oximeter server @@ -440,6 +683,7 @@ impl Config { } } +/// Arguments for running the `oximeter` collector. pub struct OximeterArguments { pub id: Uuid, pub address: SocketAddrV6, @@ -447,7 +691,7 @@ pub struct OximeterArguments { /// A server used to collect metrics from components in the control plane. pub struct Oximeter { - _agent: Arc, + agent: Arc, server: HttpServer>, } @@ -572,7 +816,67 @@ impl Oximeter { .expect("Expected an infinite retry loop contacting Nexus"); info!(log, "oximeter registered with nexus"; "id" => ?agent.id); - Ok(Self { _agent: agent, server }) + Ok(Self { agent, server }) + } + + /// Create a new `oximeter` collector running in standalone mode. + pub async fn new_standalone( + log: &Logger, + args: &OximeterArguments, + nexus: SocketAddr, + clickhouse: Option, + ) -> Result { + let db_config = clickhouse.map(DbConfig::with_address); + let agent = Arc::new( + OximeterAgent::new_standalone(args.id, db_config, &log).await?, + ); + + let dropshot_log = log.new(o!("component" => "dropshot")); + let server = HttpServerStarter::new( + &ConfigDropshot { + bind_address: SocketAddr::V6(args.address), + ..Default::default() + }, + oximeter_api(), + Arc::clone(&agent), + &dropshot_log, + ) + .map_err(|e| Error::Server(e.to_string()))? + .start(); + info!(log, "started oximeter standalone server"); + + // Notify the standalone nexus. + let client = reqwest::Client::new(); + let notify_nexus = || async { + debug!(log, "contacting nexus"); + client + .post(format!("http://{}/metrics/collectors", nexus)) + .json(&nexus_client::types::OximeterInfo { + address: server.local_addr().to_string(), + collector_id: agent.id, + }) + .send() + .await + .map_err(|e| backoff::BackoffError::transient(e.to_string()))? + .error_for_status() + .map_err(|e| backoff::BackoffError::transient(e.to_string())) + }; + let log_notification_failure = |error, delay| { + warn!( + log, + "failed to contact nexus, will retry in {:?}", delay; + "error" => ?error + ); + }; + backoff::retry_notify( + backoff::retry_policy_internal_service(), + notify_nexus, + log_notification_failure, + ) + .await + .expect("Expected an infinite retry loop contacting Nexus"); + + Ok(Self { agent, server }) } /// Serve requests forever, consuming the server. @@ -592,6 +896,20 @@ impl Oximeter { pub async fn force_collect(&self) { self.server.app_private().force_collection().await } + + /// List producers. + pub async fn list_producers( + &self, + start: Option, + limit: usize, + ) -> Vec { + self.agent.list_producers(start, limit).await + } + + /// Delete a producer by ID, stopping its collection task. + pub async fn delete_producer(&self, id: Uuid) -> Result<(), Error> { + self.agent.delete_producer(id).await + } } // Build the HTTP API internal to the control plane @@ -599,6 +917,12 @@ pub fn oximeter_api() -> ApiDescription> { let mut api = ApiDescription::new(); api.register(producers_post) .expect("Could not register producers_post API handler"); + api.register(producers_list) + .expect("Could not register producers_list API handler"); + api.register(producer_delete) + .expect("Could not register producers_delete API handler"); + api.register(collector_info) + .expect("Could not register collector_info API handler"); api } @@ -616,6 +940,79 @@ async fn producers_post( agent .register_producer(producer_info) .await - .map_err(|e| HttpError::for_internal_error(e.to_string()))?; - Ok(HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + .map(|_| HttpResponseUpdatedNoContent()) +} + +// Parameters for paginating the list of producers. +#[derive(Clone, Copy, Debug, Deserialize, schemars::JsonSchema, Serialize)] +struct ProducerPage { + id: Uuid, +} + +// List all producers +#[endpoint { + method = GET, + path = "/producers", +}] +async fn producers_list( + request_context: RequestContext>, + query: Query>, +) -> Result>, HttpError> { + let agent = request_context.context(); + let pagination = query.into_inner(); + let limit = request_context.page_limit(&pagination)?.get() as usize; + let start = match &pagination.page { + WhichPage::First(..) => None, + WhichPage::Next(ProducerPage { id }) => Some(*id), + }; + let producers = agent.list_producers(start, limit).await; + ResultsPage::new( + producers, + &EmptyScanParams {}, + |info: &ProducerEndpoint, _| ProducerPage { id: info.id }, + ) + .map(HttpResponseOk) +} + +#[derive(Clone, Copy, Debug, Deserialize, schemars::JsonSchema, Serialize)] +struct ProducerIdPathParams { + producer_id: Uuid, +} + +// Delete a producer by ID. +#[endpoint { + method = DELETE, + path = "/producers/{producer_id}", +}] +async fn producer_delete( + request_context: RequestContext>, + path: dropshot::Path, +) -> Result { + let agent = request_context.context(); + let producer_id = path.into_inner().producer_id; + agent + .delete_producer(producer_id) + .await + .map_err(HttpError::from) + .map(|_| HttpResponseDeleted()) +} + +#[derive(Clone, Copy, Debug, Deserialize, schemars::JsonSchema, Serialize)] +pub struct CollectorInfo { + /// The collector's UUID. + pub id: Uuid, +} + +// Return identifying information about this collector +#[endpoint { + method = GET, + path = "/info", +}] +async fn collector_info( + request_context: RequestContext>, +) -> Result, HttpError> { + let agent = request_context.context(); + let info = CollectorInfo { id: agent.id }; + Ok(HttpResponseOk(info)) } diff --git a/oximeter/collector/src/standalone.rs b/oximeter/collector/src/standalone.rs new file mode 100644 index 0000000000..826a5f4663 --- /dev/null +++ b/oximeter/collector/src/standalone.rs @@ -0,0 +1,263 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Implementation of a standalone fake Nexus, simply for registering producers +//! and collectors with one another. + +// Copyright 2023 Oxide Computer Company + +use crate::Error; +use dropshot::endpoint; +use dropshot::ApiDescription; +use dropshot::ConfigDropshot; +use dropshot::HttpError; +use dropshot::HttpResponseUpdatedNoContent; +use dropshot::HttpServer; +use dropshot::HttpServerStarter; +use dropshot::RequestContext; +use dropshot::TypedBody; +use nexus_types::internal_api::params::OximeterInfo; +use omicron_common::api::internal::nexus::ProducerEndpoint; +use omicron_common::FileKv; +use oximeter_client::Client; +use rand::seq::IteratorRandom; +use slog::debug; +use slog::error; +use slog::info; +use slog::o; +use slog::Drain; +use slog::Level; +use slog::Logger; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::sync::Mutex; +use uuid::Uuid; + +// An assignment of a producer to an oximeter collector. +#[derive(Debug)] +struct ProducerAssignment { + producer: ProducerEndpoint, + collector_id: Uuid, +} + +#[derive(Debug)] +struct Inner { + // Map of producers by ID to their information and assigned oximeter + // collector. + producers: HashMap, + // Map of available oximeter collectors. + collectors: HashMap, +} + +impl Inner { + fn random_collector(&self) -> Option<(Uuid, OximeterInfo)> { + self.collectors + .iter() + .choose(&mut rand::thread_rng()) + .map(|(id, info)| (*id, *info)) + } +} + +// A stripped-down Nexus server, with only the APIs for registering metric +// producers and collectors. +#[derive(Debug)] +pub struct StandaloneNexus { + pub log: Logger, + inner: Mutex, +} + +impl StandaloneNexus { + fn new(log: Logger) -> Self { + Self { + log, + inner: Mutex::new(Inner { + producers: HashMap::new(), + collectors: HashMap::new(), + }), + } + } + + async fn register_producer( + &self, + info: &ProducerEndpoint, + ) -> Result<(), HttpError> { + let mut inner = self.inner.lock().await; + let assignment = match inner.producers.get_mut(&info.id) { + None => { + // There is no record for this producer. + // + // Select a random collector, and assign it to the producer. + // We'll return the assignment from this match block. + let Some((collector_id, collector_info)) = + inner.random_collector() + else { + return Err(HttpError::for_unavail( + None, + String::from("No collectors available"), + )); + }; + let client = Client::new( + format!("http://{}", collector_info.address).as_str(), + self.log.clone(), + ); + client.producers_post(&info.into()).await.map_err(|e| { + HttpError::for_internal_error(e.to_string()) + })?; + let assignment = + ProducerAssignment { producer: info.clone(), collector_id }; + assignment + } + Some(existing_assignment) => { + // We have a record, first check if it matches the assignment we + // have. + if &existing_assignment.producer == info { + return Ok(()); + } + + // This appears to be a re-registration, e.g., the producer + // changed its IP address. Re-register it with the collector to + // which it's already assigned. + let collector_id = existing_assignment.collector_id; + let collector_info = + inner.collectors.get(&collector_id).unwrap(); + let client = Client::new( + format!("http://{}", collector_info.address).as_str(), + self.log.clone(), + ); + client.producers_post(&info.into()).await.map_err(|e| { + HttpError::for_internal_error(e.to_string()) + })?; + ProducerAssignment { producer: info.clone(), collector_id } + } + }; + inner.producers.insert(info.id, assignment); + Ok(()) + } + + async fn register_collector( + &self, + info: OximeterInfo, + ) -> Result<(), HttpError> { + // If this is being registered again, send all its assignments again. + let mut inner = self.inner.lock().await; + if inner.collectors.insert(info.collector_id, info).is_some() { + let client = Client::new( + format!("http://{}", info.address).as_str(), + self.log.clone(), + ); + for producer_info in + inner.producers.values().filter_map(|assignment| { + if assignment.collector_id == info.collector_id { + Some(&assignment.producer) + } else { + None + } + }) + { + client.producers_post(&producer_info.into()).await.map_err( + |e| HttpError::for_internal_error(e.to_string()), + )?; + } + } + Ok(()) + } +} + +// Build the HTTP API of the fake Nexus for registration. +pub fn standalone_nexus_api() -> ApiDescription> { + let mut api = ApiDescription::new(); + api.register(cpapi_producers_post) + .expect("Could not register cpapi_producers_post API handler"); + api.register(cpapi_collectors_post) + .expect("Could not register cpapi_collectors_post API handler"); + api +} + +/// Accept a registration from a new metric producer +#[endpoint { + method = POST, + path = "/metrics/producers", + }] +async fn cpapi_producers_post( + request_context: RequestContext>, + producer_info: TypedBody, +) -> Result { + let context = request_context.context(); + let producer_info = producer_info.into_inner(); + context + .register_producer(&producer_info) + .await + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(|e| HttpError::for_internal_error(e.to_string())) +} + +/// Accept a notification of a new oximeter collection server. +#[endpoint { + method = POST, + path = "/metrics/collectors", + }] +async fn cpapi_collectors_post( + request_context: RequestContext>, + oximeter_info: TypedBody, +) -> Result { + let context = request_context.context(); + let oximeter_info = oximeter_info.into_inner(); + context + .register_collector(oximeter_info) + .await + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(|e| HttpError::for_internal_error(e.to_string())) +} + +/// A standalone Nexus server, with APIs only for registering metric collectors +/// and producers. +pub struct Server { + server: HttpServer>, +} + +impl Server { + /// Create a new server listening on the provided address. + pub fn new(address: SocketAddr, log_level: Level) -> Result { + let decorator = slog_term::TermDecorator::new().build(); + let drain = slog_term::FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let drain = slog::LevelFilter::new(drain, log_level).fuse(); + let (drain, registration) = slog_dtrace::with_drain(drain); + let log = slog::Logger::root(drain.fuse(), o!(FileKv)); + if let slog_dtrace::ProbeRegistration::Failed(e) = registration { + let msg = format!("failed to register DTrace probes: {}", e); + error!(log, "{}", msg); + return Err(Error::Server(msg)); + } else { + debug!(log, "registered DTrace probes"); + } + + let nexus = Arc::new(StandaloneNexus::new( + log.new(slog::o!("component" => "nexus-standalone")), + )); + let server = HttpServerStarter::new( + &ConfigDropshot { bind_address: address, ..Default::default() }, + standalone_nexus_api(), + Arc::clone(&nexus), + &log, + ) + .map_err(|e| Error::Server(e.to_string()))? + .start(); + info!( + log, + "created standalone nexus server for metric collections"; + "address" => %address, + ); + Ok(Self { server }) + } + + pub fn log(&self) -> &Logger { + &self.server.app_private().log + } + + pub fn local_addr(&self) -> SocketAddr { + self.server.local_addr() + } +} diff --git a/oximeter/collector/tests/output/cmd-oximeter-noargs-stderr b/oximeter/collector/tests/output/cmd-oximeter-noargs-stderr index 7b736fe8a1..3f0fd4726d 100644 --- a/oximeter/collector/tests/output/cmd-oximeter-noargs-stderr +++ b/oximeter/collector/tests/output/cmd-oximeter-noargs-stderr @@ -3,9 +3,11 @@ See README.adoc for more information Usage: oximeter Commands: - openapi Print the external OpenAPI Spec document and exit - run Start an Oximeter server - help Print this message or the help of the given subcommand(s) + openapi Print the external OpenAPI Spec document and exit + run Start an Oximeter server + standalone Run `oximeter` in standalone mode for development + standalone-openapi Print the fake Nexus's standalone API + help Print this message or the help of the given subcommand(s) Options: -h, --help Print help diff --git a/oximeter/producer/Cargo.toml b/oximeter/producer/Cargo.toml index 3f74ba753f..ef2f16c8ad 100644 --- a/oximeter/producer/Cargo.toml +++ b/oximeter/producer/Cargo.toml @@ -20,3 +20,7 @@ tokio.workspace = true thiserror.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true + +[dev-dependencies] +anyhow.workspace = true +clap.workspace = true diff --git a/oximeter/producer/examples/producer.rs b/oximeter/producer/examples/producer.rs index 9ff30032ca..dd9722c80a 100644 --- a/oximeter/producer/examples/producer.rs +++ b/oximeter/producer/examples/producer.rs @@ -6,14 +6,17 @@ // Copyright 2023 Oxide Computer Company +use anyhow::Context; use chrono::DateTime; use chrono::Utc; +use clap::Parser; use dropshot::ConfigDropshot; use dropshot::ConfigLogging; use dropshot::ConfigLoggingLevel; use dropshot::HandlerTaskMode; use omicron_common::api::internal::nexus::ProducerEndpoint; use oximeter::types::Cumulative; +use oximeter::types::ProducerRegistry; use oximeter::types::Sample; use oximeter::Metric; use oximeter::MetricsError; @@ -22,9 +25,22 @@ use oximeter::Target; use oximeter_producer::Config; use oximeter_producer::LogConfig; use oximeter_producer::Server; +use std::net::SocketAddr; use std::time::Duration; use uuid::Uuid; +/// Run an example oximeter metric producer. +#[derive(Parser)] +struct Args { + /// The address to use for the producer server. + #[arg(long, default_value = "[::1]:0")] + address: SocketAddr, + + /// The address of nexus at which to register. + #[arg(long, default_value = "[::1]:12221")] + nexus: SocketAddr, +} + /// Example target describing a virtual machine. #[derive(Debug, Clone, Target)] pub struct VirtualMachine { @@ -93,30 +109,29 @@ impl Producer for CpuBusyProducer { } #[tokio::main] -async fn main() { - let address = "[::1]:0".parse().unwrap(); +async fn main() -> anyhow::Result<()> { + let args = Args::parse(); let dropshot = ConfigDropshot { - bind_address: address, + bind_address: args.address, request_body_max_bytes: 2048, default_handler_task_mode: HandlerTaskMode::Detached, }; let log = LogConfig::Config(ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug, }); + let registry = ProducerRegistry::new(); + let producer = CpuBusyProducer::new(4); + registry.register_producer(producer).unwrap(); let server_info = ProducerEndpoint { - id: Uuid::new_v4(), - address, + id: registry.producer_id(), + address: args.address, base_route: "/collect".to_string(), interval: Duration::from_secs(10), }; - let config = Config { - server_info, - registration_address: "[::1]:12221".parse().unwrap(), - dropshot, - log, - }; - let server = Server::start(&config).await.unwrap(); - let producer = CpuBusyProducer::new(4); - server.registry().register_producer(producer).unwrap(); - server.serve_forever().await.unwrap(); + let config = + Config { server_info, registration_address: args.nexus, dropshot, log }; + let server = Server::with_registry(registry, &config) + .await + .context("failed to create producer")?; + server.serve_forever().await.context("server failed") } diff --git a/oximeter/producer/src/lib.rs b/oximeter/producer/src/lib.rs index 01910af8e8..2354f9c217 100644 --- a/oximeter/producer/src/lib.rs +++ b/oximeter/producer/src/lib.rs @@ -40,6 +40,9 @@ pub enum Error { #[error("Error registering as metric producer: {0}")] RegistrationError(String), + + #[error("Producer registry and config UUIDs do not match")] + UuidMismatch, } /// Either configuration for building a logger, or an actual logger already @@ -82,14 +85,59 @@ impl Server { /// Start a new metric server, registering it with the chosen endpoint, and listening for /// requests on the associated address and route. pub async fn start(config: &Config) -> Result { - // Clone mutably, as we may update the address after the server starts, see below. - let mut config = config.clone(); + Self::with_registry( + ProducerRegistry::with_id(config.server_info.id), + &config, + ) + .await + } + + /// Create a new metric producer server, with an existing registry. + pub async fn with_registry( + registry: ProducerRegistry, + config: &Config, + ) -> Result { + Self::new_impl( + registry, + config.server_info.clone(), + &config.registration_address, + &config.dropshot, + &config.log, + ) + .await + } + + /// Serve requests for metrics. + pub async fn serve_forever(self) -> Result<(), Error> { + self.server.await.map_err(Error::Server) + } + + /// Close the server + pub async fn close(self) -> Result<(), Error> { + self.server.close().await.map_err(Error::Server) + } + + /// Return the [`ProducerRegistry`] managed by this server. + /// + /// The registry is thread-safe and clonable, so the returned reference can be used throughout + /// an application to register types implementing the [`Producer`](oximeter::traits::Producer) + /// trait. The samples generated by the registered producers will be included in response to a + /// request on the collection endpoint. + pub fn registry(&self) -> &ProducerRegistry { + &self.registry + } + + /// Return the server's local listening address + pub fn address(&self) -> std::net::SocketAddr { + self.server.local_addr() + } + fn build_logger(log: &LogConfig) -> Result { // Build a logger, either using the configuration or actual logger // provided. First build the base logger from the configuration or a // clone of the provided logger, and then add the DTrace and Dropshot // loggers on top of it. - let base_logger = match config.log { + let base_logger = match log { LogConfig::Config(conf) => conf .to_logger("metric-server") .map_err(|msg| Error::Server(msg.to_string()))?, @@ -104,74 +152,64 @@ impl Server { } else { debug!(log, "registered DTrace probes"); } - let dropshot_log = log.new(o!("component" => "dropshot")); + Ok(log) + } - // Build the producer registry and server that uses it as its context. - let registry = ProducerRegistry::with_id(config.server_info.id); - let server = HttpServerStarter::new( - &config.dropshot, + fn build_dropshot_server( + log: &Logger, + registry: &ProducerRegistry, + dropshot: &ConfigDropshot, + ) -> Result, Error> { + let dropshot_log = log.new(o!("component" => "dropshot")); + HttpServerStarter::new( + dropshot, metric_server_api(), registry.clone(), &dropshot_log, ) - .map_err(|e| Error::Server(e.to_string()))? - .start(); - - // Client code may decide to assign a specific address and/or port, or to listen on any - // available address and port, assigned by the OS. For example, `[::1]:0` would assign any - // port on localhost. If needed, update the address in the `ProducerEndpoint` with the - // actual address the server has bound. - // - // TODO-robustness: Is there a better way to do this? We'd like to support users picking an - // exact address or using whatever's available. The latter is useful during tests or other - // situations in which we don't know which ports are available. - if config.server_info.address != server.local_addr() { - assert_eq!(config.server_info.address.port(), 0); + .map_err(|e| Error::Server(e.to_string())) + .map(HttpServerStarter::start) + } + + // Create a new server registering with Nexus. + async fn new_impl( + registry: ProducerRegistry, + mut server_info: ProducerEndpoint, + registration_address: &SocketAddr, + dropshot: &ConfigDropshot, + log: &LogConfig, + ) -> Result { + if registry.producer_id() != server_info.id { + return Err(Error::UuidMismatch); + } + let log = Self::build_logger(log)?; + let server = Self::build_dropshot_server(&log, ®istry, dropshot)?; + + // Update the producer endpoint address with the actual server's + // address, to handle cases where client listens on any available + // address. + if server_info.address != server.local_addr() { + assert_eq!(server_info.address.port(), 0); debug!( log, "Requested any available port, Dropshot server has been bound to {}", server.local_addr(), ); - config.server_info.address = server.local_addr(); + server_info.address = server.local_addr(); } debug!(log, "registering metric server as a producer"); - register(config.registration_address, &log, &config.server_info) - .await?; + register(*registration_address, &log, &server_info).await?; info!( log, - "starting oximeter metric server"; - "route" => config.server_info.collection_route(), + "starting oximeter metric producer server"; + "route" => server_info.collection_route(), "producer_id" => ?registry.producer_id(), - "address" => config.server_info.address, + "address" => server.local_addr(), + "interval" => ?server_info.interval, ); Ok(Self { registry, server }) } - - /// Serve requests for metrics. - pub async fn serve_forever(self) -> Result<(), Error> { - self.server.await.map_err(Error::Server) - } - - /// Close the server - pub async fn close(self) -> Result<(), Error> { - self.server.close().await.map_err(Error::Server) - } - - /// Return the [`ProducerRegistry`] managed by this server. - /// - /// The registry is thread-safe and clonable, so the returned reference can be used throughout - /// an application to register types implementing the [`Producer`](oximeter::traits::Producer) - /// trait. The samples generated by the registered producers will be included in response to a - /// request on the collection endpoint. - pub fn registry(&self) -> &ProducerRegistry { - &self.registry - } - - /// Return the server's local listening address - pub fn address(&self) -> std::net::SocketAddr { - self.server.local_addr() - } } // Register API endpoints of the `Server`. From bb4e0cc64814d8ed6b43bfc20301abd0adad5b5c Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Wed, 4 Oct 2023 18:00:49 -0700 Subject: [PATCH 21/89] Update Propolis and Crucible to latest (#4195) Crucible updates all Crucible connections should set TCP_NODELAY (#983) Use a fixed size for tag and nonce (#957) Log crucible opts on start, order crutest options (#974) Lock the Downstairs less (#966) Cache dirty flag locally, reducing SQLite operations (#970) Make stats mutex synchronous (#961) Optimize requeue during flow control conditions (#962) Update Rust crate base64 to 0.21.4 (#950) Do less in control (#949) Fix --flush-per-blocks (#959) Fast dependency checking (#916) Update actions/checkout action to v4 (#960) Use `cargo hakari` for better workspace deps (#956) Update actions/checkout digest to 8ade135 (#939) Cache block size in Guest (#947) Update Rust crate ringbuffer to 0.15.0 (#954) Update Rust crate toml to 0.8 (#955) Update Rust crate reedline to 0.24.0 (#953) Update Rust crate libc to 0.2.148 (#952) Update Rust crate indicatif to 0.17.7 (#951) Remove unused async (#943) Use a synchronous mutex for bw/iop_tokens (#946) Make flush ID non-locking (#945) Use `oneshot` channels instead of `mpsc` for notification (#918) Use a strong type for upstairs negotiation (#941) Add a "dynamometer" option to crucible-downstairs (#931) Get new work and active count in one lock (#938) A bunch of misc test cleanup stuff (#937) Wait for a snapshot to finish on all downstairs (#920) dsc and clippy cleanup. (#935) No need to sort ackable_work (#934) Use a strong type for repair ID (#928) Keep new jobs sorted (#929) Remove state_count function on Downstairs (#927) Small cleanup to IOStateCount (#932) let cmon and IOStateCount use ClientId (#930) Fast return for zero length IOs (#926) Use a strong type for client ID (#925) A few Crucible Agent fixes (#922) Use a newtype for `JobId` (#919) Don't pass MutexGuard into functions (#917) Crutest updates, rename tests, new options (#911) Propolis updates Update tungstenite crates to 0.20 Use `strum` crate for enum-related utilities Wire up bits for CPUID customization PHD: improve artifact store (#529) Revert abort-on-panic in 'dev' cargo profile --------- Co-authored-by: Alan Hanson --- Cargo.lock | 220 ++++++++++++++++++++----------- Cargo.toml | 14 +- package-manifest.toml | 12 +- sled-agent/src/sim/sled_agent.rs | 10 +- workspace-hack/Cargo.toml | 32 +++-- 5 files changed, 182 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b931918b9c..27a165c307 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.10", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.3" @@ -474,20 +485,20 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ "bhyve_api_sys", "libc", - "num_enum 0.5.11", + "strum", ] [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ "libc", - "num_enum 0.5.11", + "strum", ] [[package]] @@ -1211,6 +1222,18 @@ dependencies = [ "libc", ] +[[package]] +name = "cpuid_profile_config" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +dependencies = [ + "propolis", + "serde", + "serde_derive", + "thiserror", + "toml 0.7.8", +] + [[package]] name = "crc" version = "3.0.1" @@ -1413,7 +1436,7 @@ dependencies = [ [[package]] name = "crucible" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=aeb69dda26c7e1a8b6eada425670cd4b83f91c07#aeb69dda26c7e1a8b6eada425670cd4b83f91c07" +source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" dependencies = [ "aes-gcm-siv", "anyhow", @@ -1447,17 +1470,18 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-util", - "toml 0.7.8", + "toml 0.8.0", "tracing", "usdt", "uuid", "version_check", + "workspace-hack", ] [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=aeb69dda26c7e1a8b6eada425670cd4b83f91c07#aeb69dda26c7e1a8b6eada425670cd4b83f91c07" +source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" dependencies = [ "anyhow", "chrono", @@ -1467,24 +1491,26 @@ dependencies = [ "schemars", "serde", "serde_json", + "workspace-hack", ] [[package]] name = "crucible-client-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=aeb69dda26c7e1a8b6eada425670cd4b83f91c07#aeb69dda26c7e1a8b6eada425670cd4b83f91c07" +source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" dependencies = [ "base64 0.21.4", "schemars", "serde", "serde_json", "uuid", + "workspace-hack", ] [[package]] name = "crucible-common" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=aeb69dda26c7e1a8b6eada425670cd4b83f91c07#aeb69dda26c7e1a8b6eada425670cd4b83f91c07" +source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" dependencies = [ "anyhow", "atty", @@ -1502,16 +1528,17 @@ dependencies = [ "tempfile", "thiserror", "tokio-rustls", - "toml 0.7.8", + "toml 0.8.0", "twox-hash", "uuid", "vergen", + "workspace-hack", ] [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=aeb69dda26c7e1a8b6eada425670cd4b83f91c07#aeb69dda26c7e1a8b6eada425670cd4b83f91c07" +source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" dependencies = [ "anyhow", "chrono", @@ -1522,32 +1549,36 @@ dependencies = [ "serde", "serde_json", "uuid", + "workspace-hack", ] [[package]] name = "crucible-protocol" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=aeb69dda26c7e1a8b6eada425670cd4b83f91c07#aeb69dda26c7e1a8b6eada425670cd4b83f91c07" +source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" dependencies = [ "anyhow", "bincode", "bytes", "crucible-common", "num_enum 0.7.0", + "schemars", "serde", "tokio-util", "uuid", + "workspace-hack", ] [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=aeb69dda26c7e1a8b6eada425670cd4b83f91c07#aeb69dda26c7e1a8b6eada425670cd4b83f91c07" +source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" dependencies = [ "libc", "num-derive", "num-traits", "thiserror", + "workspace-hack", ] [[package]] @@ -1985,10 +2016,10 @@ checksum = "7e1a8646b2c125eeb9a84ef0faa6d2d102ea0d5da60b824ade2743263117b848" [[package]] name = "dladm" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ "libc", - "num_enum 0.5.11", + "strum", ] [[package]] @@ -2322,26 +2353,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "enum-iterator" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" -dependencies = [ - "enum-iterator-derive", -] - -[[package]] -name = "enum-iterator-derive" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - [[package]] name = "env_logger" version = "0.9.3" @@ -2965,6 +2976,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.6", +] [[package]] name = "hashbrown" @@ -2972,7 +2986,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash", + "ahash 0.8.3", ] [[package]] @@ -2981,7 +2995,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ - "ahash", + "ahash 0.8.3", "allocator-api2", ] @@ -5372,6 +5386,7 @@ dependencies = [ "futures", "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -5379,11 +5394,13 @@ dependencies = [ "gateway-messages", "generic-array", "getrandom 0.2.10", + "hashbrown 0.12.3", "hashbrown 0.13.2", "hashbrown 0.14.0", "hex", "hyper", "hyper-rustls", + "indexmap 1.9.3", "indexmap 2.0.0", "inout", "ipnetwork", @@ -5401,7 +5418,9 @@ dependencies = [ "num-traits", "once_cell", "openapiv3", + "parking_lot 0.12.1", "petgraph", + "phf_shared 0.11.2", "postgres-types", "ppv-lite86", "predicates 3.0.3", @@ -5435,6 +5454,7 @@ dependencies = [ "toml_datetime", "toml_edit 0.19.15", "tracing", + "tracing-core", "trust-dns-proto", "unicode-bidi", "unicode-normalization", @@ -6568,7 +6588,7 @@ dependencies = [ [[package]] name = "propolis" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ "anyhow", "bhyve_api", @@ -6583,7 +6603,6 @@ dependencies = [ "lazy_static", "libc", "nexus-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "num_enum 0.5.11", "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", "propolis_types", "rfb", @@ -6591,6 +6610,7 @@ dependencies = [ "serde_arrays", "serde_json", "slog", + "strum", "thiserror", "tokio", "usdt", @@ -6601,7 +6621,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ "async-trait", "base64 0.21.4", @@ -6618,14 +6638,14 @@ dependencies = [ "slog", "thiserror", "tokio", - "tokio-tungstenite 0.17.2", + "tokio-tungstenite 0.20.1", "uuid", ] [[package]] name = "propolis-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ "anyhow", "async-trait", @@ -6640,7 +6660,6 @@ dependencies = [ "const_format", "crucible-client-types", "dropshot", - "enum-iterator", "erased-serde", "futures", "http", @@ -6648,7 +6667,6 @@ dependencies = [ "internal-dns 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", "lazy_static", "nexus-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "num_enum 0.5.11", "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", "oximeter-producer 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", @@ -6666,9 +6684,10 @@ dependencies = [ "slog-bunyan", "slog-dtrace", "slog-term", + "strum", "thiserror", "tokio", - "tokio-tungstenite 0.17.2", + "tokio-tungstenite 0.20.1", "tokio-util", "toml 0.7.8", "usdt", @@ -6678,8 +6697,9 @@ dependencies = [ [[package]] name = "propolis-server-config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ + "cpuid_profile_config", "serde", "serde_derive", "thiserror", @@ -6689,7 +6709,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ "schemars", "serde", @@ -7156,9 +7176,9 @@ dependencies = [ [[package]] name = "ringbuffer" -version = "0.14.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eba9638e96ac5a324654f8d47fb71c5e21abef0f072740ed9c1d4b0801faa37" +checksum = "3df6368f71f205ff9c33c076d170dd56ebf68e8161c733c0caa07a7a5509ed53" [[package]] name = "ron" @@ -7920,17 +7940,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "sha-1" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "digest", -] - [[package]] name = "sha1" version = "0.10.5" @@ -9031,26 +9040,26 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.17.3", + "tungstenite 0.18.0", ] [[package]] name = "tokio-tungstenite" -version = "0.18.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.18.0", + "tungstenite 0.20.1", ] [[package]] @@ -9218,6 +9227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", + "valuable", ] [[package]] @@ -9399,9 +9409,9 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" dependencies = [ "base64 0.13.1", "byteorder", @@ -9410,7 +9420,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "sha-1", + "sha1", "thiserror", "url", "utf-8", @@ -9418,13 +9428,13 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.18.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" dependencies = [ - "base64 0.13.1", "byteorder", "bytes", + "data-encoding", "http", "httparse", "log", @@ -9706,6 +9716,12 @@ dependencies = [ "serde", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" @@ -9740,17 +9756,16 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "viona_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ "libc", - "num_enum 0.5.11", "viona_api_sys", ] [[package]] name = "viona_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=de6369aa45a255f896da0a3ddd2b7152c036a4e9#de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" dependencies = [ "libc", ] @@ -10329,6 +10344,61 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "workspace-hack" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +dependencies = [ + "bitflags 2.4.0", + "bytes", + "cc", + "chrono", + "console", + "crossbeam-utils", + "crypto-common", + "digest", + "either", + "futures-channel", + "futures-core", + "futures-executor", + "futures-sink", + "futures-util", + "getrandom 0.2.10", + "hashbrown 0.12.3", + "hex", + "hyper", + "indexmap 1.9.3", + "libc", + "log", + "mio", + "num-traits", + "once_cell", + "openapiv3", + "parking_lot 0.12.1", + "phf_shared 0.11.2", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_core 0.6.4", + "reqwest", + "rustls", + "schemars", + "semver 1.0.18", + "serde", + "slog", + "syn 1.0.109", + "syn 2.0.32", + "time", + "time-macros", + "tokio", + "tokio-util", + "toml_datetime", + "toml_edit 0.19.15", + "tracing", + "tracing-core", + "usdt", + "uuid", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index fb610128ed..2af44b5559 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,10 +161,10 @@ cookie = "0.16" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "aeb69dda26c7e1a8b6eada425670cd4b83f91c07" } -crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "aeb69dda26c7e1a8b6eada425670cd4b83f91c07" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "aeb69dda26c7e1a8b6eada425670cd4b83f91c07" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "aeb69dda26c7e1a8b6eada425670cd4b83f91c07" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" } +crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" } curve25519-dalek = "4" datatest-stable = "0.1.3" display-error-chain = "0.1.1" @@ -277,9 +277,9 @@ pretty-hex = "0.3.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "de6369aa45a255f896da0a3ddd2b7152c036a4e9" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "de6369aa45a255f896da0a3ddd2b7152c036a4e9", features = [ "generated-migration" ] } -propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "de6369aa45a255f896da0a3ddd2b7152c036a4e9", default-features = false, features = ["mock-only"] } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "42c878b71a58d430dfc306126af5d40ca816d70f" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "42c878b71a58d430dfc306126af5d40ca816d70f", features = [ "generated-migration" ] } +propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "42c878b71a58d430dfc306126af5d40ca816d70f", default-features = false, features = ["mock-only"] } proptest = "1.2.0" quote = "1.0" rand = "0.8.5" diff --git a/package-manifest.toml b/package-manifest.toml index c776f6d96d..ff229e5def 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -381,10 +381,10 @@ only_for_targets.image = "standard" # 3. Use source.type = "manual" instead of "prebuilt" source.type = "prebuilt" source.repo = "crucible" -source.commit = "aeb69dda26c7e1a8b6eada425670cd4b83f91c07" +source.commit = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible.sha256.txt -source.sha256 = "3845327bde9df585ee8771c85eefc3e63a48981f14298d5fca62f4f6fe25c917" +source.sha256 = "0671570dfed8bff8e64c42a41269d961426bdd07e72b9ca8c2e3f28e7ead3c1c" output.type = "zone" [package.crucible-pantry] @@ -392,10 +392,10 @@ service_name = "crucible_pantry" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "aeb69dda26c7e1a8b6eada425670cd4b83f91c07" +source.commit = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-pantry.sha256.txt -source.sha256 = "a3f2fc92d9ae184a66c402dfe33b1d1c128f356d6be70671de421be600d4064a" +source.sha256 = "c35cc24945d047f8d77e438ee606e6a83be64f0f97356fdc3308be716dcf3718" output.type = "zone" # Refer to @@ -406,10 +406,10 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "de6369aa45a255f896da0a3ddd2b7152c036a4e9" +source.commit = "42c878b71a58d430dfc306126af5d40ca816d70f" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "182597a153793096826992f499a94be54c746e346a3566802e1fe7e78b2ccf2f" +source.sha256 = "dce4d82bb936e990262abcaa279eee7e33a19930880b23f49fa3851cded18567" output.type = "zone" [package.maghemite] diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index e53295f823..42fff355a5 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -617,14 +617,8 @@ impl SledAgent { ..Default::default() }; let propolis_log = log.new(o!("component" => "propolis-server-mock")); - let config = propolis_server::config::Config { - bootrom: Default::default(), - pci_bridges: Default::default(), - chipset: Default::default(), - devices: Default::default(), - block_devs: Default::default(), - }; - let private = Arc::new(PropolisContext::new(config, propolis_log)); + let private = + Arc::new(PropolisContext::new(Default::default(), propolis_log)); info!(log, "Starting mock propolis-server..."); let dropshot_log = log.new(o!("component" => "dropshot")); let mock_api = propolis_server::mock_server::api(); diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 820b2d2336..8854ef27bc 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -39,6 +39,7 @@ flate2 = { version = "1.0.27" } futures = { version = "0.3.28" } futures-channel = { version = "0.3.28", features = ["sink"] } futures-core = { version = "0.3.28" } +futures-executor = { version = "0.3.28" } futures-io = { version = "0.3.28", default-features = false, features = ["std"] } futures-sink = { version = "0.3.28" } futures-task = { version = "0.3.28", default-features = false, features = ["std"] } @@ -48,9 +49,11 @@ generic-array = { version = "0.14.7", default-features = false, features = ["mor getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } +hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12.3", features = ["raw"] } hex = { version = "0.4.3", features = ["serde"] } hyper = { version = "0.14.27", features = ["full"] } -indexmap = { version = "2.0.0", features = ["serde"] } +indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1.9.3", default-features = false, features = ["serde-1", "std"] } +indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2.0.0", features = ["serde"] } inout = { version = "0.1.3", default-features = false, features = ["std"] } ipnetwork = { version = "0.20.0", features = ["schemars"] } itertools = { version = "0.10.5" } @@ -65,11 +68,13 @@ num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } openapiv3 = { version = "1.0.3", default-features = false, features = ["skip_serializing_defaults"] } +parking_lot = { version = "0.12.1", features = ["send_guard"] } petgraph = { version = "0.6.4", features = ["serde-1"] } +phf_shared = { version = "0.11.2" } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.0.3" } -rand = { version = "0.8.5", features = ["min_const_gen"] } +rand = { version = "0.8.5", features = ["min_const_gen", "small_rng"] } rand_chacha = { version = "0.3.1" } regex = { version = "1.9.5" } regex-automata = { version = "0.3.8", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } @@ -86,7 +91,7 @@ slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "rele spin = { version = "0.9.8" } string_cache = { version = "0.8.7" } subtle = { version = "2.5.0" } -syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] } +syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } textwrap = { version = "0.16.0" } time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } @@ -94,9 +99,8 @@ tokio = { version = "1.32.0", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } -toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } -toml_edit = { version = "0.19.15", features = ["serde"] } tracing = { version = "0.1.37", features = ["log"] } +tracing-core = { version = "0.1.31" } trust-dns-proto = { version = "0.22.0" } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } @@ -133,6 +137,7 @@ flate2 = { version = "1.0.27" } futures = { version = "0.3.28" } futures-channel = { version = "0.3.28", features = ["sink"] } futures-core = { version = "0.3.28" } +futures-executor = { version = "0.3.28" } futures-io = { version = "0.3.28", default-features = false, features = ["std"] } futures-sink = { version = "0.3.28" } futures-task = { version = "0.3.28", default-features = false, features = ["std"] } @@ -142,9 +147,11 @@ generic-array = { version = "0.14.7", default-features = false, features = ["mor getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } +hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12.3", features = ["raw"] } hex = { version = "0.4.3", features = ["serde"] } hyper = { version = "0.14.27", features = ["full"] } -indexmap = { version = "2.0.0", features = ["serde"] } +indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1.9.3", default-features = false, features = ["serde-1", "std"] } +indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2.0.0", features = ["serde"] } inout = { version = "0.1.3", default-features = false, features = ["std"] } ipnetwork = { version = "0.20.0", features = ["schemars"] } itertools = { version = "0.10.5" } @@ -159,11 +166,13 @@ num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } openapiv3 = { version = "1.0.3", default-features = false, features = ["skip_serializing_defaults"] } +parking_lot = { version = "0.12.1", features = ["send_guard"] } petgraph = { version = "0.6.4", features = ["serde-1"] } +phf_shared = { version = "0.11.2" } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.0.3" } -rand = { version = "0.8.5", features = ["min_const_gen"] } +rand = { version = "0.8.5", features = ["min_const_gen", "small_rng"] } rand_chacha = { version = "0.3.1" } regex = { version = "1.9.5" } regex-automata = { version = "0.3.8", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } @@ -180,7 +189,7 @@ slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "rele spin = { version = "0.9.8" } string_cache = { version = "0.8.7" } subtle = { version = "2.5.0" } -syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] } +syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } textwrap = { version = "0.16.0" } time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } @@ -189,9 +198,8 @@ tokio = { version = "1.32.0", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } -toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } -toml_edit = { version = "0.19.15", features = ["serde"] } tracing = { version = "0.1.37", features = ["log"] } +tracing-core = { version = "0.1.31" } trust-dns-proto = { version = "0.22.0" } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } @@ -250,6 +258,8 @@ hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } +toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } +toml_edit = { version = "0.19.15", features = ["serde"] } [target.x86_64-unknown-illumos.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } @@ -257,5 +267,7 @@ hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } +toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } +toml_edit = { version = "0.19.15", features = ["serde"] } ### END HAKARI SECTION From 9aabe2aee4bbfe9af1cb9424a93cf6a59f13b9a6 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 5 Oct 2023 09:46:26 -0700 Subject: [PATCH 22/89] Add transaction retry to schema upgrade integration tests (#4209) Fixes https://github.com/oxidecomputer/omicron/issues/4207 --- nexus/tests/integration_tests/schema.rs | 51 ++++++++++++++++++++----- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 2c62f156e1..1d4556e8ed 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -62,6 +62,47 @@ async fn test_setup<'a>( builder } +// Attempts to apply an update as a transaction. +// +// Only returns an error if the transaction failed to commit. +async fn apply_update_as_transaction_inner( + client: &omicron_test_utils::dev::db::Client, + sql: &str, +) -> Result<(), tokio_postgres::Error> { + client.batch_execute("BEGIN;").await.expect("Failed to BEGIN transaction"); + client.batch_execute(&sql).await.expect("Failed to execute update"); + client.batch_execute("COMMIT;").await?; + Ok(()) +} + +// Applies an update as a transaction. +// +// Automatically retries transactions that can be retried client-side. +async fn apply_update_as_transaction( + log: &Logger, + client: &omicron_test_utils::dev::db::Client, + sql: &str, +) { + loop { + match apply_update_as_transaction_inner(client, sql).await { + Ok(()) => break, + Err(err) => { + client + .batch_execute("ROLLBACK;") + .await + .expect("Failed to ROLLBACK failed transaction"); + if let Some(code) = err.code() { + if code == &tokio_postgres::error::SqlState::T_R_SERIALIZATION_FAILURE { + warn!(log, "Transaction retrying"); + continue; + } + } + panic!("Failed to apply update: {err}"); + } + } + } +} + async fn apply_update( log: &Logger, crdb: &CockroachInstance, @@ -87,15 +128,7 @@ async fn apply_update( for _ in 0..times_to_apply { for sql in sqls.iter() { - client - .batch_execute("BEGIN;") - .await - .expect("Failed to BEGIN update"); - client.batch_execute(&sql).await.expect("Failed to execute update"); - client - .batch_execute("COMMIT;") - .await - .expect("Failed to COMMIT update"); + apply_update_as_transaction(log, &client, sql).await; } } From 6cf8181ba678855e3f131ad2914e90d06de02ac3 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Thu, 5 Oct 2023 10:28:21 -0700 Subject: [PATCH 23/89] top-level cleanup: move `thing-flinger` into `dev-tools` (#4213) --- Cargo.toml | 4 ++-- {deploy => dev-tools/thing-flinger}/.gitignore | 0 {deploy => dev-tools/thing-flinger}/Cargo.toml | 0 {deploy => dev-tools/thing-flinger}/README.adoc | 0 .../thing-flinger}/src/bin/deployment-example.toml | 0 {deploy => dev-tools/thing-flinger}/src/bin/thing-flinger.rs | 0 6 files changed, 2 insertions(+), 2 deletions(-) rename {deploy => dev-tools/thing-flinger}/.gitignore (100%) rename {deploy => dev-tools/thing-flinger}/Cargo.toml (100%) rename {deploy => dev-tools/thing-flinger}/README.adoc (100%) rename {deploy => dev-tools/thing-flinger}/src/bin/deployment-example.toml (100%) rename {deploy => dev-tools/thing-flinger}/src/bin/thing-flinger.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 2af44b5559..29291e8a19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,9 +8,9 @@ members = [ "common", "crdb-seed", "ddm-admin-client", - "deploy", "dev-tools/omdb", "dev-tools/omicron-dev", + "dev-tools/thing-flinger", "dev-tools/xtask", "dns-server", "dns-service-client", @@ -75,9 +75,9 @@ default-members = [ "common", "ddm-admin-client", "dpd-client", - "deploy", "dev-tools/omdb", "dev-tools/omicron-dev", + "dev-tools/thing-flinger", "dev-tools/xtask", "dns-server", "dns-service-client", diff --git a/deploy/.gitignore b/dev-tools/thing-flinger/.gitignore similarity index 100% rename from deploy/.gitignore rename to dev-tools/thing-flinger/.gitignore diff --git a/deploy/Cargo.toml b/dev-tools/thing-flinger/Cargo.toml similarity index 100% rename from deploy/Cargo.toml rename to dev-tools/thing-flinger/Cargo.toml diff --git a/deploy/README.adoc b/dev-tools/thing-flinger/README.adoc similarity index 100% rename from deploy/README.adoc rename to dev-tools/thing-flinger/README.adoc diff --git a/deploy/src/bin/deployment-example.toml b/dev-tools/thing-flinger/src/bin/deployment-example.toml similarity index 100% rename from deploy/src/bin/deployment-example.toml rename to dev-tools/thing-flinger/src/bin/deployment-example.toml diff --git a/deploy/src/bin/thing-flinger.rs b/dev-tools/thing-flinger/src/bin/thing-flinger.rs similarity index 100% rename from deploy/src/bin/thing-flinger.rs rename to dev-tools/thing-flinger/src/bin/thing-flinger.rs From ba291b8ab2293eb3e4cdf85a1bae072d75343b5e Mon Sep 17 00:00:00 2001 From: Jordan Hendricks Date: Thu, 5 Oct 2023 14:06:04 -0700 Subject: [PATCH 24/89] Add example of using an SSH tunnel to access the console in development deployments (#4200) --- docs/how-to-run.adoc | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index aa1ee3c73d..04d274da8b 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -143,7 +143,10 @@ $ svcadm enable ipfilter Other network configurations are possible but beyond the scope of this doc. -When making this choice, note that **in order to use the system once it's set up, you will need to be able to access it from a web browser.** If you go with option 2 here, you may need to use an ssh tunnel or the like to do this. +When making this choice, note that **in order to use the system once it's set +up, you will need to be able to access it from a web browser.** If you go with +option 2 here, you may need to use an SSH tunnel (see: +<>) or the like to do this. === Picking a "machine" type @@ -433,7 +436,32 @@ Where did 192.168.1.20 come from? That's the external address of the external DNS server. We knew that because it's listed in the `external_dns_ips` entry of the `config-rss.toml` file we're using. -Having looked this up, the easiest thing will be to use `http://192.168.1.21` for your URL (replacing with `https` if you used a certificate, and replacing that IP if needed). If you've set up networking right, you should be able to reach this from your web browser. You may have to instruct the browser to accept a self-signed TLS certificate. See also <<_connecting_securely_with_tls_using_the_cli>>. +Having looked this up, the easiest thing will be to use `http://192.168.1.21` for your URL (replacing with `https` if you used a certificate, and replacing that IP if needed). If you've set up networking right, you should be able to reach this from your web browser. You may have to instruct the browser to accept a self-signed TLS certificate. See also <>. + +=== Setting up an SSH tunnel for console access + +If you set up a fake external network (method 2 in <>), one +way to be able to access the console of your deployment is by setting up an SSH +tunnel. Console access is required to use the CLI for device authentication. +The following is an example of how to access the console with an SSH tunnel. + +Nexus serves the console, so first get a nexus IP from the instructions above. + +In this example, Omicron is running on the lab machine `dunkin`. Usually, you'll +want to set up the tunnel from the machine where you run a browser, to the +machine running Omicron. In this example, one would run this on the machine +running the browser: + +``` +$ ssh -L 1234:192.168.1.22:80 dunkin.eng.oxide.computer +``` + +The above command configures `ssh` to bind to the TCP port `1234` on the machine +running the browser, forward packets through the ssh connection, and redirect +them to 192.168.1.22 port 80 *as seen from the other side of the connection*. + +Now you should be able to access the console from the browser on this machine, +via something like: `127.0.0.1:1234`, using the port from the `ssh` command. === Using the CLI From a2bb889cd21aeef7c287ee2da469a771bc684c01 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Thu, 5 Oct 2023 14:29:43 -0700 Subject: [PATCH 25/89] top-level cleanup: consolidate clients (#4214) --- Cargo.toml | 78 +++++++++---------- .../bootstrap-agent-client}/Cargo.toml | 0 .../bootstrap-agent-client}/src/lib.rs | 2 +- .../ddm-admin-client}/Cargo.toml | 0 .../ddm-admin-client}/build.rs | 14 ++-- .../ddm-admin-client}/src/lib.rs | 0 .../dns-service-client}/Cargo.toml | 0 .../dns-service-client}/src/lib.rs | 2 +- {dpd-client => clients/dpd-client}/Cargo.toml | 0 {dpd-client => clients/dpd-client}/build.rs | 18 ++--- {dpd-client => clients/dpd-client}/src/lib.rs | 0 .../gateway-client}/Cargo.toml | 0 .../gateway-client}/src/lib.rs | 2 +- .../installinator-artifact-client}/Cargo.toml | 0 .../installinator-artifact-client}/src/lib.rs | 2 +- .../nexus-client}/Cargo.toml | 0 .../nexus-client}/src/lib.rs | 2 +- .../oxide-client}/Cargo.toml | 0 .../oxide-client}/src/lib.rs | 2 +- .../oximeter-client}/Cargo.toml | 0 .../oximeter-client}/src/lib.rs | 2 +- .../sled-agent-client}/Cargo.toml | 0 .../sled-agent-client}/src/lib.rs | 2 +- .../wicketd-client}/Cargo.toml | 0 .../wicketd-client}/src/lib.rs | 2 +- 25 files changed, 64 insertions(+), 64 deletions(-) rename {bootstrap-agent-client => clients/bootstrap-agent-client}/Cargo.toml (100%) rename {bootstrap-agent-client => clients/bootstrap-agent-client}/src/lib.rs (97%) rename {ddm-admin-client => clients/ddm-admin-client}/Cargo.toml (100%) rename {ddm-admin-client => clients/ddm-admin-client}/build.rs (87%) rename {ddm-admin-client => clients/ddm-admin-client}/src/lib.rs (100%) rename {dns-service-client => clients/dns-service-client}/Cargo.toml (100%) rename {dns-service-client => clients/dns-service-client}/src/lib.rs (98%) rename {dpd-client => clients/dpd-client}/Cargo.toml (100%) rename {dpd-client => clients/dpd-client}/build.rs (87%) rename {dpd-client => clients/dpd-client}/src/lib.rs (100%) rename {gateway-client => clients/gateway-client}/Cargo.toml (100%) rename {gateway-client => clients/gateway-client}/src/lib.rs (98%) rename {installinator-artifact-client => clients/installinator-artifact-client}/Cargo.toml (100%) rename {installinator-artifact-client => clients/installinator-artifact-client}/src/lib.rs (96%) rename {nexus-client => clients/nexus-client}/Cargo.toml (100%) rename {nexus-client => clients/nexus-client}/src/lib.rs (99%) rename {oxide-client => clients/oxide-client}/Cargo.toml (100%) rename {oxide-client => clients/oxide-client}/src/lib.rs (99%) rename {oximeter-client => clients/oximeter-client}/Cargo.toml (100%) rename {oximeter-client => clients/oximeter-client}/src/lib.rs (93%) rename {sled-agent-client => clients/sled-agent-client}/Cargo.toml (100%) rename {sled-agent-client => clients/sled-agent-client}/src/lib.rs (99%) rename {wicketd-client => clients/wicketd-client}/Cargo.toml (100%) rename {wicketd-client => clients/wicketd-client}/src/lib.rs (99%) diff --git a/Cargo.toml b/Cargo.toml index 29291e8a19..1ca8a02886 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,26 +2,31 @@ members = [ "api_identity", "bootstore", - "bootstrap-agent-client", "caboose-util", "certificates", + "clients/bootstrap-agent-client", + "clients/ddm-admin-client", + "clients/dns-service-client", + "clients/dpd-client", + "clients/gateway-client", + "clients/installinator-artifact-client", + "clients/nexus-client", + "clients/oxide-client", + "clients/oximeter-client", + "clients/sled-agent-client", + "clients/wicketd-client", "common", "crdb-seed", - "ddm-admin-client", "dev-tools/omdb", "dev-tools/omicron-dev", "dev-tools/thing-flinger", "dev-tools/xtask", "dns-server", - "dns-service-client", - "dpd-client", "end-to-end-tests", "gateway-cli", - "gateway-client", "gateway-test-utils", "gateway", "illumos-utils", - "installinator-artifact-client", "installinator-artifactd", "installinator-common", "installinator", @@ -29,7 +34,6 @@ members = [ "internal-dns", "ipcc-key-value", "key-manager", - "nexus-client", "nexus", "nexus/authz-macros", "nexus/db-macros", @@ -40,8 +44,6 @@ members = [ "nexus/test-utils-macros", "nexus/test-utils", "nexus/types", - "oxide-client", - "oximeter-client", "oximeter/collector", "oximeter/db", "oximeter/instruments", @@ -51,7 +53,6 @@ members = [ "package", "passwords", "rpaths", - "sled-agent-client", "sled-agent", "sled-hardware", "sp-sim", @@ -62,70 +63,69 @@ members = [ "wicket-common", "wicket-dbg", "wicket", - "wicketd-client", "wicketd", "workspace-hack", ] default-members = [ - "bootstrap-agent-client", "bootstore", "caboose-util", "certificates", + "clients/bootstrap-agent-client", + "clients/ddm-admin-client", + "clients/dns-service-client", + "clients/dpd-client", + "clients/gateway-client", + "clients/installinator-artifact-client", + "clients/nexus-client", + "clients/oxide-client", + "clients/oximeter-client", + "clients/sled-agent-client", + "clients/wicketd-client", "common", - "ddm-admin-client", - "dpd-client", "dev-tools/omdb", "dev-tools/omicron-dev", "dev-tools/thing-flinger", "dev-tools/xtask", "dns-server", - "dns-service-client", - "gateway", "gateway-cli", - "gateway-client", "gateway-test-utils", + "gateway", "illumos-utils", - "installinator", - "installinator-artifact-client", "installinator-artifactd", "installinator-common", - "internal-dns", + "installinator", "internal-dns-cli", + "internal-dns", "ipcc-key-value", "key-manager", "nexus", - "nexus-client", "nexus/authz-macros", "nexus/db-macros", "nexus/db-model", "nexus/db-queries", "nexus/defaults", "nexus/types", - "oxide-client", - "oximeter-client", "oximeter/collector", "oximeter/db", "oximeter/instruments", - "oximeter/oximeter", "oximeter/oximeter-macro-impl", + "oximeter/oximeter", "oximeter/producer", "package", "passwords", "rpaths", "sled-agent", - "sled-agent-client", "sled-hardware", "sp-sim", "test-utils", - "tufaceous", "tufaceous-lib", + "tufaceous", "update-engine", - "wicket", "wicket-common", "wicket-dbg", + "wicket", "wicketd", - "wicketd-client", ] resolver = "2" @@ -144,7 +144,7 @@ bb8 = "0.8.1" bcs = "0.1.5" bincode = "1.3.3" bootstore = { path = "bootstore" } -bootstrap-agent-client = { path = "bootstrap-agent-client" } +bootstrap-agent-client = { path = "clients/bootstrap-agent-client" } buf-list = { version = "1.0.3", features = ["tokio1"] } byteorder = "1.4.3" bytes = "1.5.0" @@ -168,7 +168,7 @@ crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "20273 curve25519-dalek = "4" datatest-stable = "0.1.3" display-error-chain = "0.1.1" -ddm-admin-client = { path = "ddm-admin-client" } +ddm-admin-client = { path = "clients/ddm-admin-client" } db-macros = { path = "nexus/db-macros" } debug-ignore = "1.0.5" derive_more = "0.99.17" @@ -176,8 +176,8 @@ derive-where = "1.2.5" diesel = { version = "2.1.1", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } diesel-dtrace = { git = "https://github.com/oxidecomputer/diesel-dtrace", branch = "main" } dns-server = { path = "dns-server" } -dns-service-client = { path = "dns-service-client" } -dpd-client = { path = "dpd-client" } +dns-service-client = { path = "clients/dns-service-client" } +dpd-client = { path = "clients/dpd-client" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } either = "1.9.0" expectorate = "1.1.0" @@ -187,7 +187,7 @@ flume = "0.11.0" foreign-types = "0.3.2" fs-err = "2.9.0" futures = "0.3.28" -gateway-client = { path = "gateway-client" } +gateway-client = { path = "clients/gateway-client" } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", default-features = false, features = ["std"] } gateway-sp-comms = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d" } gateway-test-utils = { path = "gateway-test-utils" } @@ -209,7 +209,7 @@ indexmap = "2.0.0" indicatif = { version = "0.17.6", features = ["rayon"] } installinator = { path = "installinator" } installinator-artifactd = { path = "installinator-artifactd" } -installinator-artifact-client = { path = "installinator-artifact-client" } +installinator-artifact-client = { path = "clients/installinator-artifact-client" } installinator-common = { path = "installinator-common" } internal-dns = { path = "internal-dns" } ipcc-key-value = { path = "ipcc-key-value" } @@ -223,7 +223,7 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } mime_guess = "2.0.4" mockall = "0.11" newtype_derive = "0.1.6" -nexus-client = { path = "nexus-client" } +nexus-client = { path = "clients/nexus-client" } nexus-db-model = { path = "nexus/db-model" } nexus-db-queries = { path = "nexus/db-queries" } nexus-defaults = { path = "nexus/defaults" } @@ -244,7 +244,7 @@ omicron-rpaths = { path = "rpaths" } omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.8.3" -oxide-client = { path = "oxide-client" } +oxide-client = { path = "clients/oxide-client" } oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "98d33125413f01722947e322f82caf9d22209434", features = [ "api", "std" ] } once_cell = "1.18.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } @@ -257,7 +257,7 @@ opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "98d33125413 oso = "0.26" owo-colors = "3.5.0" oximeter = { path = "oximeter/oximeter" } -oximeter-client = { path = "oximeter-client" } +oximeter-client = { path = "clients/oximeter-client" } oximeter-db = { path = "oximeter/db/" } oximeter-collector = { path = "oximeter/collector" } oximeter-instruments = { path = "oximeter/instruments" } @@ -315,7 +315,7 @@ signal-hook = "0.3" signal-hook-tokio = { version = "0.3", features = [ "futures-v0_3" ] } similar-asserts = "1.5.0" sled = "0.34" -sled-agent-client = { path = "sled-agent-client" } +sled-agent-client = { path = "clients/sled-agent-client" } sled-hardware = { path = "sled-hardware" } slog = { version = "2.7", features = [ "dynamic-keys", "max_level_trace", "release_max_level_debug" ] } slog-async = "2.8" @@ -370,7 +370,7 @@ usdt = "0.3" walkdir = "2.4" wicket = { path = "wicket" } wicket-common = { path = "wicket-common" } -wicketd-client = { path = "wicketd-client" } +wicketd-client = { path = "clients/wicketd-client" } zeroize = { version = "1.6.0", features = ["zeroize_derive", "std"] } zip = { version = "0.6.6", default-features = false, features = ["deflate","bzip2"] } zone = { version = "0.2", default-features = false, features = ["async"] } diff --git a/bootstrap-agent-client/Cargo.toml b/clients/bootstrap-agent-client/Cargo.toml similarity index 100% rename from bootstrap-agent-client/Cargo.toml rename to clients/bootstrap-agent-client/Cargo.toml diff --git a/bootstrap-agent-client/src/lib.rs b/clients/bootstrap-agent-client/src/lib.rs similarity index 97% rename from bootstrap-agent-client/src/lib.rs rename to clients/bootstrap-agent-client/src/lib.rs index 5a159e299a..3f8b20e1f5 100644 --- a/bootstrap-agent-client/src/lib.rs +++ b/clients/bootstrap-agent-client/src/lib.rs @@ -5,7 +5,7 @@ //! Interface for making API requests to a Bootstrap Agent progenitor::generate_api!( - spec = "../openapi/bootstrap-agent.json", + spec = "../../openapi/bootstrap-agent.json", inner_type = slog::Logger, pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { slog::debug!(log, "client request"; diff --git a/ddm-admin-client/Cargo.toml b/clients/ddm-admin-client/Cargo.toml similarity index 100% rename from ddm-admin-client/Cargo.toml rename to clients/ddm-admin-client/Cargo.toml diff --git a/ddm-admin-client/build.rs b/clients/ddm-admin-client/build.rs similarity index 87% rename from ddm-admin-client/build.rs rename to clients/ddm-admin-client/build.rs index ef4183fee3..e3c1345eda 100644 --- a/ddm-admin-client/build.rs +++ b/clients/ddm-admin-client/build.rs @@ -16,23 +16,23 @@ use std::path::Path; fn main() -> Result<()> { // Find the current maghemite repo commit from our package manifest. - let manifest = fs::read_to_string("../package-manifest.toml") - .context("failed to read ../package-manifest.toml")?; - println!("cargo:rerun-if-changed=../package-manifest.toml"); + let manifest = fs::read_to_string("../../package-manifest.toml") + .context("failed to read ../../package-manifest.toml")?; + println!("cargo:rerun-if-changed=../../package-manifest.toml"); let config: Config = toml::from_str(&manifest) - .context("failed to parse ../package-manifest.toml")?; + .context("failed to parse ../../package-manifest.toml")?; let maghemite = config .packages .get("maghemite") - .context("missing maghemite package in ../package-manifest.toml")?; + .context("missing maghemite package in ../../package-manifest.toml")?; let local_path = match &maghemite.source { PackageSource::Prebuilt { commit, .. } => { // Report a relatively verbose error if we haven't downloaded the requisite // openapi spec. let local_path = - format!("../out/downloads/ddm-admin-{commit}.json"); + format!("../../out/downloads/ddm-admin-{commit}.json"); if !Path::new(&local_path).exists() { bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); } @@ -42,7 +42,7 @@ fn main() -> Result<()> { PackageSource::Manual => { let local_path = - "../out/downloads/ddm-admin-manual.json".to_string(); + "../../out/downloads/ddm-admin-manual.json".to_string(); if !Path::new(&local_path).exists() { bail!("{local_path} doesn't exist, please copy manually built ddm-admin.json there!"); } diff --git a/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs similarity index 100% rename from ddm-admin-client/src/lib.rs rename to clients/ddm-admin-client/src/lib.rs diff --git a/dns-service-client/Cargo.toml b/clients/dns-service-client/Cargo.toml similarity index 100% rename from dns-service-client/Cargo.toml rename to clients/dns-service-client/Cargo.toml diff --git a/dns-service-client/src/lib.rs b/clients/dns-service-client/src/lib.rs similarity index 98% rename from dns-service-client/src/lib.rs rename to clients/dns-service-client/src/lib.rs index 9b729b1c5c..931e68322f 100644 --- a/dns-service-client/src/lib.rs +++ b/clients/dns-service-client/src/lib.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. progenitor::generate_api!( - spec = "../openapi/dns-server.json", + spec = "../../openapi/dns-server.json", inner_type = slog::Logger, derives = [schemars::JsonSchema, Eq, PartialEq], pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { diff --git a/dpd-client/Cargo.toml b/clients/dpd-client/Cargo.toml similarity index 100% rename from dpd-client/Cargo.toml rename to clients/dpd-client/Cargo.toml diff --git a/dpd-client/build.rs b/clients/dpd-client/build.rs similarity index 87% rename from dpd-client/build.rs rename to clients/dpd-client/build.rs index 2aaa8437e7..6a65ab9495 100644 --- a/dpd-client/build.rs +++ b/clients/dpd-client/build.rs @@ -22,23 +22,23 @@ use std::path::Path; fn main() -> Result<()> { // Find the current dendrite repo commit from our package manifest. - let manifest = fs::read_to_string("../package-manifest.toml") - .context("failed to read ../package-manifest.toml")?; - println!("cargo:rerun-if-changed=../package-manifest.toml"); + let manifest = fs::read_to_string("../../package-manifest.toml") + .context("failed to read ../../package-manifest.toml")?; + println!("cargo:rerun-if-changed=../../package-manifest.toml"); let config: Config = toml::from_str(&manifest) - .context("failed to parse ../package-manifest.toml")?; + .context("failed to parse ../../package-manifest.toml")?; let dendrite = config .packages .get("dendrite-asic") - .context("missing dendrite package in ../package-manifest.toml")?; + .context("missing dendrite package in ../../package-manifest.toml")?; let local_path = match &dendrite.source { PackageSource::Prebuilt { commit, .. } => { - // Report a relatively verbose error if we haven't downloaded the requisite - // openapi spec. - let local_path = format!("../out/downloads/dpd-{commit}.json"); + // Report a relatively verbose error if we haven't downloaded the + // requisite openapi spec. + let local_path = format!("../../out/downloads/dpd-{commit}.json"); if !Path::new(&local_path).exists() { bail!("{local_path} doesn't exist; rerun `tools/ci_download_dendrite_openapi` (after updating `tools/dendrite_openapi_version` if the dendrite commit in package-manifest.toml has changed)"); } @@ -47,7 +47,7 @@ fn main() -> Result<()> { } PackageSource::Manual => { - let local_path = "../out/downloads/dpd-manual.json".to_string(); + let local_path = "../../out/downloads/dpd-manual.json".to_string(); if !Path::new(&local_path).exists() { bail!("{local_path} doesn't exist, please copy manually built dpd.json there!"); } diff --git a/dpd-client/src/lib.rs b/clients/dpd-client/src/lib.rs similarity index 100% rename from dpd-client/src/lib.rs rename to clients/dpd-client/src/lib.rs diff --git a/gateway-client/Cargo.toml b/clients/gateway-client/Cargo.toml similarity index 100% rename from gateway-client/Cargo.toml rename to clients/gateway-client/Cargo.toml diff --git a/gateway-client/src/lib.rs b/clients/gateway-client/src/lib.rs similarity index 98% rename from gateway-client/src/lib.rs rename to clients/gateway-client/src/lib.rs index 7992eff9e4..800254b197 100644 --- a/gateway-client/src/lib.rs +++ b/clients/gateway-client/src/lib.rs @@ -34,7 +34,7 @@ // it is no longer useful to directly expose the JsonSchema types, we can go // back to reusing `omicron_common`. progenitor::generate_api!( - spec = "../openapi/gateway.json", + spec = "../../openapi/gateway.json", inner_type = slog::Logger, pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { slog::debug!(log, "client request"; diff --git a/installinator-artifact-client/Cargo.toml b/clients/installinator-artifact-client/Cargo.toml similarity index 100% rename from installinator-artifact-client/Cargo.toml rename to clients/installinator-artifact-client/Cargo.toml diff --git a/installinator-artifact-client/src/lib.rs b/clients/installinator-artifact-client/src/lib.rs similarity index 96% rename from installinator-artifact-client/src/lib.rs rename to clients/installinator-artifact-client/src/lib.rs index aa5ceb863a..de3072a34a 100644 --- a/installinator-artifact-client/src/lib.rs +++ b/clients/installinator-artifact-client/src/lib.rs @@ -5,7 +5,7 @@ //! Interface for making API requests to installinator-artifactd. progenitor::generate_api!( - spec = "../openapi/installinator-artifactd.json", + spec = "../../openapi/installinator-artifactd.json", inner_type = slog::Logger, pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { slog::debug!(log, "client request"; diff --git a/nexus-client/Cargo.toml b/clients/nexus-client/Cargo.toml similarity index 100% rename from nexus-client/Cargo.toml rename to clients/nexus-client/Cargo.toml diff --git a/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs similarity index 99% rename from nexus-client/src/lib.rs rename to clients/nexus-client/src/lib.rs index e5cec83f39..412ca70497 100644 --- a/nexus-client/src/lib.rs +++ b/clients/nexus-client/src/lib.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; progenitor::generate_api!( - spec = "../openapi/nexus-internal.json", + spec = "../../openapi/nexus-internal.json", derives = [schemars::JsonSchema, PartialEq], inner_type = slog::Logger, pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { diff --git a/oxide-client/Cargo.toml b/clients/oxide-client/Cargo.toml similarity index 100% rename from oxide-client/Cargo.toml rename to clients/oxide-client/Cargo.toml diff --git a/oxide-client/src/lib.rs b/clients/oxide-client/src/lib.rs similarity index 99% rename from oxide-client/src/lib.rs rename to clients/oxide-client/src/lib.rs index 7d34697002..07a190c38e 100644 --- a/oxide-client/src/lib.rs +++ b/clients/oxide-client/src/lib.rs @@ -16,7 +16,7 @@ use trust_dns_resolver::config::{ use trust_dns_resolver::TokioAsyncResolver; progenitor::generate_api!( - spec = "../openapi/nexus.json", + spec = "../../openapi/nexus.json", interface = Builder, tags = Separate, ); diff --git a/oximeter-client/Cargo.toml b/clients/oximeter-client/Cargo.toml similarity index 100% rename from oximeter-client/Cargo.toml rename to clients/oximeter-client/Cargo.toml diff --git a/oximeter-client/src/lib.rs b/clients/oximeter-client/src/lib.rs similarity index 93% rename from oximeter-client/src/lib.rs rename to clients/oximeter-client/src/lib.rs index 9f326fdee8..7bd17d7e76 100644 --- a/oximeter-client/src/lib.rs +++ b/clients/oximeter-client/src/lib.rs @@ -6,7 +6,7 @@ //! Interface for API requests to an Oximeter metric collection server -omicron_common::generate_logging_api!("../openapi/oximeter.json"); +omicron_common::generate_logging_api!("../../openapi/oximeter.json"); impl omicron_common::api::external::ClientError for types::Error { fn message(&self) -> String { diff --git a/sled-agent-client/Cargo.toml b/clients/sled-agent-client/Cargo.toml similarity index 100% rename from sled-agent-client/Cargo.toml rename to clients/sled-agent-client/Cargo.toml diff --git a/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs similarity index 99% rename from sled-agent-client/src/lib.rs rename to clients/sled-agent-client/src/lib.rs index 98e7f207e3..68e60e8d95 100644 --- a/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -9,7 +9,7 @@ use omicron_common::generate_logging_api; use std::convert::TryFrom; use uuid::Uuid; -generate_logging_api!("../openapi/sled-agent.json"); +generate_logging_api!("../../openapi/sled-agent.json"); impl omicron_common::api::external::ClientError for types::Error { fn message(&self) -> String { diff --git a/wicketd-client/Cargo.toml b/clients/wicketd-client/Cargo.toml similarity index 100% rename from wicketd-client/Cargo.toml rename to clients/wicketd-client/Cargo.toml diff --git a/wicketd-client/src/lib.rs b/clients/wicketd-client/src/lib.rs similarity index 99% rename from wicketd-client/src/lib.rs rename to clients/wicketd-client/src/lib.rs index 3f113ea271..ff45232520 100644 --- a/wicketd-client/src/lib.rs +++ b/clients/wicketd-client/src/lib.rs @@ -5,7 +5,7 @@ //! Interface for making API requests to wicketd progenitor::generate_api!( - spec = "../openapi/wicketd.json", + spec = "../../openapi/wicketd.json", inner_type = slog::Logger, pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { slog::debug!(log, "client request"; From fb2b4a1d3286bb0c1bb6207fd1aa541037709295 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Thu, 5 Oct 2023 15:16:22 -0700 Subject: [PATCH 26/89] tools/install_opte: Freeze the package to the pinned version. (#4215) --- tools/install_opte.sh | 27 +++++++++++++++++++++++++++ tools/uninstall_opte.sh | 14 ++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/tools/install_opte.sh b/tools/install_opte.sh index f670adf163..20a33b05a5 100755 --- a/tools/install_opte.sh +++ b/tools/install_opte.sh @@ -51,6 +51,26 @@ fi # Grab the version of the opte package to install OPTE_VERSION="$(cat "$OMICRON_TOP/tools/opte_version")" +OMICRON_FROZEN_PKG_COMMENT="OMICRON-PINNED-PACKAGE" + +# Once we install, we mark the package as frozen at that particular version. +# This makes sure that a `pkg update` won't automatically move us forward +# (and hence defeat the whole point of pinning). +# But this also prevents us from installig the next version so we must +# unfreeze first. +if PKG_FROZEN=$(pkg freeze | grep driver/network/opte); then + FROZEN_COMMENT=$(echo "$PKG_FROZEN" | awk '{ print $(NF) }') + + # Compare the comment to make sure this is indeed our previous doing + if [ "$FROZEN_COMMENT" != "$OMICRON_FROZEN_PKG_COMMENT" ]; then + echo "Found driver/network/opte previously frozen but not by us:" + echo $PKG_FROZEN + exit 1 + fi + + pfexec pkg unfreeze driver/network/opte +fi + # Actually install the xde kernel module and opteadm tool RC=0 pfexec pkg install -v pkg://helios-dev/driver/network/opte@"$OPTE_VERSION" || RC=$? @@ -63,6 +83,13 @@ else exit "$RC" fi +RC=0 +pfexec pkg freeze -c "$OMICRON_FROZEN_PKG_COMMENT" driver/network/opte@"$OPTE_VERSION" || RC=$? +if [[ "$RC" -ne 0 ]]; then + echo "Failed to pin opte package to $OPTE_VERSION" + exit $RC +fi + # Check the user's path RC=0 which opteadm > /dev/null || RC=$? diff --git a/tools/uninstall_opte.sh b/tools/uninstall_opte.sh index a833d029aa..c8ee0f5b28 100755 --- a/tools/uninstall_opte.sh +++ b/tools/uninstall_opte.sh @@ -165,6 +165,19 @@ function restore_xde_and_opte { fi } +function unfreeze_opte_pkg { + OMICRON_FROZEN_PKG_COMMENT="OMICRON-PINNED-PACKAGE" + + # If we've frozen a particular version, let's be good citizens + # and clear that as well. + if PKG_FROZEN=$(pkg freeze | grep driver/network/opte); then + FROZEN_COMMENT=$(echo "$PKG_FROZEN" | awk '{ print $(NF) }') + if [ "$FROZEN_COMMENT" == "$OMICRON_FROZEN_PKG_COMMENT" ]; then + pkg unfreeze driver/network/opte + fi + fi +} + function ensure_not_already_on_helios { local RC=0 pkg list "$STOCK_CONSOLIDATION"* || RC=$? @@ -179,5 +192,6 @@ uninstall_xde_and_opte for PUBLISHER in "${PUBLISHERS[@]}"; do remove_publisher "$PUBLISHER" done +unfreeze_opte_pkg ensure_not_already_on_helios to_stock_helios "$CONSOLIDATION" From c76ca69d34729d430f2779bfabb556c33eaf9bd6 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Thu, 5 Oct 2023 18:22:25 -0500 Subject: [PATCH 27/89] [trivial] add doc comments to snapshot ID and image ID on disk response type (#4217) Noticed these fields on the disk response, wondered what they are, and was surprised to see no description in the docs. https://docs-2meozyi0z-oxidecomputer.vercel.app/api/disk_view image --- common/src/api/external/mod.rs | 2 ++ openapi/nexus.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 1d7e6884d1..91ed7e4240 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -952,7 +952,9 @@ pub struct Disk { #[serde(flatten)] pub identity: IdentityMetadata, pub project_id: Uuid, + /// ID of snapshot from which disk was created, if any pub snapshot_id: Option, + /// ID of image from which disk was created, if any pub image_id: Option, pub size: ByteCount, pub block_size: ByteCount, diff --git a/openapi/nexus.json b/openapi/nexus.json index 779b1f556c..9330b0ef47 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -9615,6 +9615,7 @@ }, "image_id": { "nullable": true, + "description": "ID of image from which disk was created, if any", "type": "string", "format": "uuid" }, @@ -9635,6 +9636,7 @@ }, "snapshot_id": { "nullable": true, + "description": "ID of snapshot from which disk was created, if any", "type": "string", "format": "uuid" }, From 230637ab5c22e4e0d6c82d5198887e2240289958 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Thu, 5 Oct 2023 19:25:47 -0700 Subject: [PATCH 28/89] `omdb` support for showing devices visible to MGS (#4162) --- Cargo.lock | 5 + clients/gateway-client/src/lib.rs | 16 +- dev-tools/omdb/Cargo.toml | 5 +- dev-tools/omdb/src/bin/omdb/main.rs | 4 + dev-tools/omdb/src/bin/omdb/mgs.rs | 488 ++++++++++++++++++ dev-tools/omdb/tests/successes.out | 105 ++++ dev-tools/omdb/tests/test_all_output.rs | 16 +- dev-tools/omdb/tests/usage_errors.out | 20 + dev-tools/omicron-dev/Cargo.toml | 2 + dev-tools/omicron-dev/src/bin/omicron-dev.rs | 38 ++ .../output/cmd-omicron-dev-noargs-stderr | 1 + 11 files changed, 697 insertions(+), 3 deletions(-) create mode 100644 dev-tools/omdb/src/bin/omdb/mgs.rs diff --git a/Cargo.lock b/Cargo.lock index 27a165c307..aad3a8782a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4951,6 +4951,8 @@ dependencies = [ "dropshot", "expectorate", "futures", + "gateway-messages", + "gateway-test-utils", "libc", "nexus-test-interface", "nexus-test-utils", @@ -5140,6 +5142,9 @@ dependencies = [ "dropshot", "expectorate", "futures", + "gateway-client", + "gateway-messages", + "gateway-test-utils", "humantime", "internal-dns 0.1.0", "ipnetwork", diff --git a/clients/gateway-client/src/lib.rs b/clients/gateway-client/src/lib.rs index 800254b197..b071d34975 100644 --- a/clients/gateway-client/src/lib.rs +++ b/clients/gateway-client/src/lib.rs @@ -48,7 +48,7 @@ progenitor::generate_api!( }), derives = [schemars::JsonSchema], patch = { - SpIdentifier = { derives = [Copy, PartialEq, Hash, Eq, PartialOrd, Ord, Serialize, Deserialize] }, + SpIdentifier = { derives = [Copy, PartialEq, Hash, Eq, Serialize, Deserialize] }, SpIgnition = { derives = [PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize] }, SpIgnitionSystemType = { derives = [Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize] }, SpState = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize] }, @@ -59,3 +59,17 @@ progenitor::generate_api!( HostPhase2RecoveryImageId = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize] }, }, ); + +// Override the impl of Ord for SpIdentifier because the default one orders the +// fields in a different order than people are likely to want. +impl Ord for crate::types::SpIdentifier { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.type_.cmp(&other.type_).then(self.slot.cmp(&other.slot)) + } +} + +impl PartialOrd for crate::types::SpIdentifier { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml index cd4af6e947..ff3c650d6d 100644 --- a/dev-tools/omdb/Cargo.toml +++ b/dev-tools/omdb/Cargo.toml @@ -14,9 +14,12 @@ chrono.workspace = true clap.workspace = true diesel.workspace = true dropshot.workspace = true +futures.workspace = true +gateway-client.workspace = true +gateway-messages.workspace = true +gateway-test-utils.workspace = true humantime.workspace = true internal-dns.workspace = true -futures.workspace = true nexus-client.workspace = true nexus-db-model.workspace = true nexus-db-queries.workspace = true diff --git a/dev-tools/omdb/src/bin/omdb/main.rs b/dev-tools/omdb/src/bin/omdb/main.rs index d1a56e1d80..32141d2809 100644 --- a/dev-tools/omdb/src/bin/omdb/main.rs +++ b/dev-tools/omdb/src/bin/omdb/main.rs @@ -41,6 +41,7 @@ use std::net::SocketAddr; use std::net::SocketAddrV6; mod db; +mod mgs; mod nexus; mod oximeter; mod sled_agent; @@ -57,6 +58,7 @@ async fn main() -> Result<(), anyhow::Error> { match &args.command { OmdbCommands::Db(db) => db.run_cmd(&args, &log).await, + OmdbCommands::Mgs(mgs) => mgs.run_cmd(&args, &log).await, OmdbCommands::Nexus(nexus) => nexus.run_cmd(&args, &log).await, OmdbCommands::Oximeter(oximeter) => oximeter.run_cmd(&log).await, OmdbCommands::SledAgent(sled) => sled.run_cmd(&args, &log).await, @@ -155,6 +157,8 @@ impl Omdb { enum OmdbCommands { /// Query the control plane database (CockroachDB) Db(db::DbArgs), + /// Debug a specific Management Gateway Service instance + Mgs(mgs::MgsArgs), /// Debug a specific Nexus instance Nexus(nexus::NexusArgs), /// Query oximeter collector state diff --git a/dev-tools/omdb/src/bin/omdb/mgs.rs b/dev-tools/omdb/src/bin/omdb/mgs.rs new file mode 100644 index 0000000000..d2938418e1 --- /dev/null +++ b/dev-tools/omdb/src/bin/omdb/mgs.rs @@ -0,0 +1,488 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Prototype code for collecting information from systems in the rack + +use crate::Omdb; +use anyhow::Context; +use clap::Args; +use clap::Subcommand; +use futures::StreamExt; +use gateway_client::types::PowerState; +use gateway_client::types::RotSlot; +use gateway_client::types::RotState; +use gateway_client::types::SpComponentCaboose; +use gateway_client::types::SpComponentInfo; +use gateway_client::types::SpIdentifier; +use gateway_client::types::SpIgnition; +use gateway_client::types::SpIgnitionInfo; +use gateway_client::types::SpIgnitionSystemType; +use gateway_client::types::SpState; +use gateway_client::types::SpType; +use tabled::Tabled; + +/// Arguments to the "omdb mgs" subcommand +#[derive(Debug, Args)] +pub struct MgsArgs { + /// URL of an MGS instance to query + #[clap(long, env("OMDB_MGS_URL"))] + mgs_url: Option, + + #[command(subcommand)] + command: MgsCommands, +} + +#[derive(Debug, Subcommand)] +enum MgsCommands { + /// Show information about devices and components visible to MGS + Inventory(InventoryArgs), +} + +#[derive(Debug, Args)] +struct InventoryArgs {} + +impl MgsArgs { + pub(crate) async fn run_cmd( + &self, + omdb: &Omdb, + log: &slog::Logger, + ) -> Result<(), anyhow::Error> { + let mgs_url = match &self.mgs_url { + Some(cli_or_env_url) => cli_or_env_url.clone(), + None => { + eprintln!( + "note: MGS URL not specified. Will pick one from DNS." + ); + let addrs = omdb + .dns_lookup_all( + log.clone(), + internal_dns::ServiceName::ManagementGatewayService, + ) + .await?; + let addr = addrs.into_iter().next().expect( + "expected at least one MGS address from \ + successful DNS lookup", + ); + format!("http://{}", addr) + } + }; + eprintln!("note: using MGS URL {}", &mgs_url); + let mgs_client = gateway_client::Client::new(&mgs_url, log.clone()); + + match &self.command { + MgsCommands::Inventory(inventory_args) => { + cmd_mgs_inventory(&mgs_client, inventory_args).await + } + } + } +} + +/// Runs `omdb mgs inventory` +/// +/// Shows devices and components that are visible to an MGS instance. +async fn cmd_mgs_inventory( + mgs_client: &gateway_client::Client, + _args: &InventoryArgs, +) -> Result<(), anyhow::Error> { + // Report all the SP identifiers that MGS is configured to talk to. + println!("ALL CONFIGURED SPs\n"); + let mut sp_ids = mgs_client + .sp_all_ids() + .await + .context("listing SP identifiers")? + .into_inner(); + sp_ids.sort(); + show_sp_ids(&sp_ids)?; + println!(""); + + // Report which SPs are visible via Ignition. + println!("SPs FOUND THROUGH IGNITION\n"); + let mut sp_list_ignition = mgs_client + .ignition_list() + .await + .context("listing ignition")? + .into_inner(); + sp_list_ignition.sort_by(|a, b| a.id.cmp(&b.id)); + show_sps_from_ignition(&sp_list_ignition)?; + println!(""); + + // Print basic state about each SP that's visible to ignition. + println!("SERVICE PROCESSOR STATES\n"); + let mgs_client = std::sync::Arc::new(mgs_client); + let c = &mgs_client; + let mut sp_infos = + futures::stream::iter(sp_list_ignition.iter().filter_map(|ignition| { + if matches!(ignition.details, SpIgnition::Yes { .. }) { + Some(ignition.id) + } else { + None + } + })) + .then(|sp_id| async move { + c.sp_get(sp_id.type_, sp_id.slot) + .await + .with_context(|| format!("fetching info about SP {:?}", sp_id)) + .map(|s| (sp_id, s)) + }) + .collect::>>() + .await + .into_iter() + .filter_map(|r| match r { + Ok((sp_id, v)) => Some((sp_id, v.into_inner())), + Err(error) => { + eprintln!("error: {:?}", error); + None + } + }) + .collect::>(); + sp_infos.sort(); + show_sp_states(&sp_infos)?; + println!(""); + + // Print detailed information about each SP that we've found so far. + for (sp_id, sp_state) in &sp_infos { + show_sp_details(&mgs_client, sp_id, sp_state).await?; + } + + Ok(()) +} + +fn sp_type_to_str(s: &SpType) -> &'static str { + match s { + SpType::Sled => "Sled", + SpType::Power => "Power", + SpType::Switch => "Switch", + } +} + +fn show_sp_ids(sp_ids: &[SpIdentifier]) -> Result<(), anyhow::Error> { + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct SpIdRow { + #[tabled(rename = "TYPE")] + type_: &'static str, + slot: u32, + } + + impl<'a> From<&'a SpIdentifier> for SpIdRow { + fn from(id: &SpIdentifier) -> Self { + SpIdRow { type_: sp_type_to_str(&id.type_), slot: id.slot } + } + } + + let table_rows = sp_ids.iter().map(SpIdRow::from); + let table = tabled::Table::new(table_rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("{}", textwrap::indent(&table.to_string(), " ")); + Ok(()) +} + +fn show_sps_from_ignition( + sp_list_ignition: &[SpIgnitionInfo], +) -> Result<(), anyhow::Error> { + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct IgnitionRow { + #[tabled(rename = "TYPE")] + type_: &'static str, + slot: u32, + system_type: String, + } + + impl<'a> From<&'a SpIgnitionInfo> for IgnitionRow { + fn from(value: &SpIgnitionInfo) -> Self { + IgnitionRow { + type_: sp_type_to_str(&value.id.type_), + slot: value.id.slot, + system_type: match value.details { + SpIgnition::No => "-".to_string(), + SpIgnition::Yes { + id: SpIgnitionSystemType::Gimlet, + .. + } => "Gimlet".to_string(), + SpIgnition::Yes { + id: SpIgnitionSystemType::Sidecar, + .. + } => "Sidecar".to_string(), + SpIgnition::Yes { + id: SpIgnitionSystemType::Psc, .. + } => "PSC".to_string(), + SpIgnition::Yes { + id: SpIgnitionSystemType::Unknown(v), + .. + } => format!("unknown: type {}", v), + }, + } + } + } + + let table_rows = sp_list_ignition.iter().map(IgnitionRow::from); + let table = tabled::Table::new(table_rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("{}", textwrap::indent(&table.to_string(), " ")); + Ok(()) +} + +fn show_sp_states( + sp_states: &[(SpIdentifier, SpState)], +) -> Result<(), anyhow::Error> { + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct SpStateRow<'a> { + #[tabled(rename = "TYPE")] + type_: &'static str, + slot: u32, + model: String, + serial: String, + rev: u32, + hubris: &'a str, + pwr: &'static str, + rot_active: String, + } + + impl<'a> From<&'a (SpIdentifier, SpState)> for SpStateRow<'a> { + fn from((id, v): &'a (SpIdentifier, SpState)) -> Self { + SpStateRow { + type_: sp_type_to_str(&id.type_), + slot: id.slot, + model: v.model.clone(), + serial: v.serial_number.clone(), + rev: v.revision, + hubris: &v.hubris_archive_id, + pwr: match v.power_state { + PowerState::A0 => "A0", + PowerState::A1 => "A1", + PowerState::A2 => "A2", + }, + rot_active: match &v.rot { + RotState::CommunicationFailed { message } => { + format!("error: {}", message) + } + RotState::Enabled { active: RotSlot::A, .. } => { + "slot A".to_string() + } + RotState::Enabled { active: RotSlot::B, .. } => { + "slot B".to_string() + } + }, + } + } + } + + let table_rows = sp_states.iter().map(SpStateRow::from); + let table = tabled::Table::new(table_rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("{}", textwrap::indent(&table.to_string(), " ")); + Ok(()) +} + +const COMPONENTS_WITH_CABOOSES: &'static [&'static str] = &["sp", "rot"]; + +async fn show_sp_details( + mgs_client: &gateway_client::Client, + sp_id: &SpIdentifier, + sp_state: &SpState, +) -> Result<(), anyhow::Error> { + println!( + "SP DETAILS: type {:?} slot {}\n", + sp_type_to_str(&sp_id.type_), + sp_id.slot + ); + + println!(" ROOT OF TRUST\n"); + match &sp_state.rot { + RotState::CommunicationFailed { message } => { + println!(" error: {}", message); + } + RotState::Enabled { + active, + pending_persistent_boot_preference, + persistent_boot_preference, + slot_a_sha3_256_digest, + slot_b_sha3_256_digest, + transient_boot_preference, + } => { + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct Row { + name: &'static str, + value: String, + } + + let rows = vec![ + Row { + name: "active slot", + value: format!("slot {:?}", active), + }, + Row { + name: "persistent boot preference", + value: format!("slot {:?}", persistent_boot_preference), + }, + Row { + name: "pending persistent boot preference", + value: pending_persistent_boot_preference + .map(|s| format!("slot {:?}", s)) + .unwrap_or_else(|| "-".to_string()), + }, + Row { + name: "transient boot preference", + value: transient_boot_preference + .map(|s| format!("slot {:?}", s)) + .unwrap_or_else(|| "-".to_string()), + }, + Row { + name: "slot A SHA3 256 digest", + value: slot_a_sha3_256_digest + .clone() + .unwrap_or_else(|| "-".to_string()), + }, + Row { + name: "slot B SHA3 256 digest", + value: slot_b_sha3_256_digest + .clone() + .unwrap_or_else(|| "-".to_string()), + }, + ]; + + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("{}", textwrap::indent(&table.to_string(), " ")); + println!(""); + } + } + + let component_list = mgs_client + .sp_component_list(sp_id.type_, sp_id.slot) + .await + .with_context(|| format!("fetching components for SP {:?}", sp_id)); + let list = match component_list { + Ok(l) => l.into_inner(), + Err(e) => { + eprintln!("error: {:#}", e); + return Ok(()); + } + }; + + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct SpComponentRow<'a> { + name: &'a str, + description: &'a str, + device: &'a str, + presence: String, + serial: String, + } + + impl<'a> From<&'a SpComponentInfo> for SpComponentRow<'a> { + fn from(v: &'a SpComponentInfo) -> Self { + SpComponentRow { + name: &v.component, + description: &v.description, + device: &v.device, + presence: format!("{:?}", v.presence), + serial: format!("{:?}", v.serial_number), + } + } + } + + if list.components.is_empty() { + println!(" COMPONENTS: none found\n"); + return Ok(()); + } + + let table_rows = list.components.iter().map(SpComponentRow::from); + let table = tabled::Table::new(table_rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!(" COMPONENTS\n"); + println!("{}", textwrap::indent(&table.to_string(), " ")); + println!(""); + + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct CabooseRow { + component: String, + board: String, + git_commit: String, + name: String, + version: String, + } + + impl<'a> From<(&'a SpIdentifier, &'a SpComponentInfo, SpComponentCaboose)> + for CabooseRow + { + fn from( + (_sp_id, component, caboose): ( + &'a SpIdentifier, + &'a SpComponentInfo, + SpComponentCaboose, + ), + ) -> Self { + CabooseRow { + component: component.component.clone(), + board: caboose.board, + git_commit: caboose.git_commit, + name: caboose.name, + version: caboose.version.unwrap_or_else(|| "-".to_string()), + } + } + } + + let mut cabooses = Vec::new(); + for c in &list.components { + if !COMPONENTS_WITH_CABOOSES.contains(&c.component.as_str()) { + continue; + } + + for i in 0..1 { + let r = mgs_client + .sp_component_caboose_get( + sp_id.type_, + sp_id.slot, + &c.component, + i, + ) + .await + .with_context(|| { + format!( + "get caboose for sp type {:?} sp slot {} \ + component {:?} slot {}", + sp_id.type_, sp_id.slot, &c.component, i + ) + }); + match r { + Ok(v) => { + cabooses.push(CabooseRow::from((sp_id, c, v.into_inner()))) + } + Err(error) => { + eprintln!("warn: {:#}", error); + } + } + } + } + + if cabooses.is_empty() { + println!(" CABOOSES: none found\n"); + return Ok(()); + } + + let table = tabled::Table::new(cabooses) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!(" COMPONENT CABOOSES\n"); + println!("{}", textwrap::indent(&table.to_string(), " ")); + println!(""); + + Ok(()) +} diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index b1464cb824..eb075a84ea 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -84,6 +84,111 @@ stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable note: database schema version matches expected (5.0.0) ============================================= +EXECUTING COMMAND: omdb ["mgs", "inventory"] +termination: Exited(0) +--------------------------------------------- +stdout: +ALL CONFIGURED SPs + + TYPE SLOT + Sled 0 + Sled 1 + Switch 0 + Switch 1 + +SPs FOUND THROUGH IGNITION + + TYPE SLOT SYSTEM_TYPE + Sled 0 Gimlet + Sled 1 Gimlet + Switch 0 Sidecar + Switch 1 Sidecar + +SERVICE PROCESSOR STATES + + TYPE SLOT MODEL SERIAL REV HUBRIS PWR ROT_ACTIVE + Sled 0 FAKE_SIM_GIMLET SimGimlet00 0 0000000000000000 A2 slot A + Sled 1 FAKE_SIM_GIMLET SimGimlet01 0 0000000000000000 A2 slot A + Switch 0 FAKE_SIM_SIDECAR SimSidecar0 0 0000000000000000 A2 slot A + Switch 1 FAKE_SIM_SIDECAR SimSidecar1 0 0000000000000000 A2 slot A + +SP DETAILS: type "Sled" slot 0 + + ROOT OF TRUST + + NAME VALUE + active slot slot A + persistent boot preference slot A + pending persistent boot preference - + transient boot preference - + slot A SHA3 256 digest - + slot B SHA3 256 digest - + + COMPONENTS + + NAME DESCRIPTION DEVICE PRESENCE SERIAL + sp3-host-cpu FAKE host cpu sp3-host-cpu Present None + dev-0 FAKE temperature sensor fake-tmp-sensor Failed None + + CABOOSES: none found + +SP DETAILS: type "Sled" slot 1 + + ROOT OF TRUST + + NAME VALUE + active slot slot A + persistent boot preference slot A + pending persistent boot preference - + transient boot preference - + slot A SHA3 256 digest - + slot B SHA3 256 digest - + + COMPONENTS + + NAME DESCRIPTION DEVICE PRESENCE SERIAL + sp3-host-cpu FAKE host cpu sp3-host-cpu Present None + + CABOOSES: none found + +SP DETAILS: type "Switch" slot 0 + + ROOT OF TRUST + + NAME VALUE + active slot slot A + persistent boot preference slot A + pending persistent boot preference - + transient boot preference - + slot A SHA3 256 digest - + slot B SHA3 256 digest - + + COMPONENTS + + NAME DESCRIPTION DEVICE PRESENCE SERIAL + dev-0 FAKE temperature sensor 1 fake-tmp-sensor Present None + dev-1 FAKE temperature sensor 2 fake-tmp-sensor Failed None + + CABOOSES: none found + +SP DETAILS: type "Switch" slot 1 + + ROOT OF TRUST + + NAME VALUE + active slot slot A + persistent boot preference slot A + pending persistent boot preference - + transient boot preference - + slot A SHA3 256 digest - + slot B SHA3 256 digest - + + COMPONENTS: none found + +--------------------------------------------- +stderr: +note: using MGS URL http://[::1]:REDACTED_PORT/ +============================================= EXECUTING COMMAND: omdb ["nexus", "background-tasks", "doc"] termination: Exited(0) --------------------------------------------- diff --git a/dev-tools/omdb/tests/test_all_output.rs b/dev-tools/omdb/tests/test_all_output.rs index d757369ead..90e93ee429 100644 --- a/dev-tools/omdb/tests/test_all_output.rs +++ b/dev-tools/omdb/tests/test_all_output.rs @@ -42,6 +42,7 @@ async fn test_omdb_usage_errors() { &["db", "dns", "names"], &["db", "services"], &["db", "network"], + &["mgs"], &["nexus"], &["nexus", "background-tasks"], &["sled-agent"], @@ -58,10 +59,16 @@ async fn test_omdb_usage_errors() { #[nexus_test] async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) { + let gwtestctx = gateway_test_utils::setup::test_setup( + "test_omdb_success_case", + gateway_messages::SpPort::One, + ) + .await; let cmd_path = path_to_executable(CMD_OMDB); let postgres_url = cptestctx.database.listen_url(); let nexus_internal_url = format!("http://{}/", cptestctx.internal_client.bind_address); + let mgs_url = format!("http://{}/", gwtestctx.client.bind_address); let mut output = String::new(); let invocations: &[&[&'static str]] = &[ &["db", "dns", "show"], @@ -70,6 +77,7 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) { &["db", "services", "list-instances"], &["db", "services", "list-by-sled"], &["db", "sleds"], + &["mgs", "inventory"], &["nexus", "background-tasks", "doc"], &["nexus", "background-tasks", "show"], // We can't easily test the sled agent output because that's only @@ -81,9 +89,14 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) { println!("running commands with args: {:?}", args); let p = postgres_url.to_string(); let u = nexus_internal_url.clone(); + let g = mgs_url.clone(); do_run( &mut output, - move |exec| exec.env("OMDB_DB_URL", &p).env("OMDB_NEXUS_URL", &u), + move |exec| { + exec.env("OMDB_DB_URL", &p) + .env("OMDB_NEXUS_URL", &u) + .env("OMDB_MGS_URL", &g) + }, &cmd_path, args, ) @@ -91,6 +104,7 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) { } assert_contents("tests/successes.out", &output); + gwtestctx.teardown().await; } /// Verify that we properly deal with cases where: diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index dc2a16bc47..7bedc3ecbc 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -10,6 +10,7 @@ Usage: omdb [OPTIONS] Commands: db Query the control plane database (CockroachDB) + mgs Debug a specific Management Gateway Service instance nexus Debug a specific Nexus instance oximeter Query oximeter collector state sled-agent Debug a specific Sled @@ -33,6 +34,7 @@ Usage: omdb [OPTIONS] Commands: db Query the control plane database (CockroachDB) + mgs Debug a specific Management Gateway Service instance nexus Debug a specific Nexus instance oximeter Query oximeter collector state sled-agent Debug a specific Sled @@ -208,6 +210,24 @@ Options: --verbose Print out raw data structures from the data store -h, --help Print help ============================================= +EXECUTING COMMAND: omdb ["mgs"] +termination: Exited(2) +--------------------------------------------- +stdout: +--------------------------------------------- +stderr: +Debug a specific Management Gateway Service instance + +Usage: omdb mgs [OPTIONS] + +Commands: + inventory Show information about devices and components visible to MGS + help Print this message or the help of the given subcommand(s) + +Options: + --mgs-url URL of an MGS instance to query [env: OMDB_MGS_URL=] + -h, --help Print help +============================================= EXECUTING COMMAND: omdb ["nexus"] termination: Exited(2) --------------------------------------------- diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml index 5439b69c76..251ee16c01 100644 --- a/dev-tools/omicron-dev/Cargo.toml +++ b/dev-tools/omicron-dev/Cargo.toml @@ -13,6 +13,8 @@ camino.workspace = true clap.workspace = true dropshot.workspace = true futures.workspace = true +gateway-messages.workspace = true +gateway-test-utils.workspace = true libc.workspace = true nexus-test-utils.workspace = true nexus-test-interface.workspace = true diff --git a/dev-tools/omicron-dev/src/bin/omicron-dev.rs b/dev-tools/omicron-dev/src/bin/omicron-dev.rs index 14617d6ba4..9107766d8a 100644 --- a/dev-tools/omicron-dev/src/bin/omicron-dev.rs +++ b/dev-tools/omicron-dev/src/bin/omicron-dev.rs @@ -30,6 +30,7 @@ async fn main() -> Result<(), anyhow::Error> { OmicronDb::DbPopulate { ref args } => cmd_db_populate(args).await, OmicronDb::DbWipe { ref args } => cmd_db_wipe(args).await, OmicronDb::ChRun { ref args } => cmd_clickhouse_run(args).await, + OmicronDb::MgsRun { ref args } => cmd_mgs_run(args).await, OmicronDb::RunAll { ref args } => cmd_run_all(args).await, OmicronDb::CertCreate { ref args } => cmd_cert_create(args).await, }; @@ -68,6 +69,12 @@ enum OmicronDb { args: ChRunArgs, }, + /// Run a simulated Management Gateway Service for development + MgsRun { + #[clap(flatten)] + args: MgsRunArgs, + }, + /// Run a full simulated control plane RunAll { #[clap(flatten)] @@ -465,3 +472,34 @@ fn write_private_file( .with_context(|| format!("open {:?} for writing", path))?; file.write_all(contents).with_context(|| format!("write to {:?}", path)) } + +#[derive(Clone, Debug, Args)] +struct MgsRunArgs {} + +async fn cmd_mgs_run(_args: &MgsRunArgs) -> Result<(), anyhow::Error> { + // Start a stream listening for SIGINT + let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); + let mut signal_stream = signals.fuse(); + + println!("omicron-dev: setting up MGS ... "); + let gwtestctx = gateway_test_utils::setup::test_setup( + "omicron-dev", + gateway_messages::SpPort::One, + ) + .await; + println!("omicron-dev: MGS is running."); + + let addr = gwtestctx.client.bind_address; + println!("omicron-dev: MGS API: http://{:?}", addr); + + // Wait for a signal. + let caught_signal = signal_stream.next().await; + assert_eq!(caught_signal.unwrap(), SIGINT); + eprintln!( + "omicron-dev: caught signal, shutting down and removing \ + temporary directory" + ); + + gwtestctx.teardown().await; + Ok(()) +} diff --git a/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr b/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr index f3c28e1ab9..ac1c87e165 100644 --- a/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr +++ b/dev-tools/omicron-dev/tests/output/cmd-omicron-dev-noargs-stderr @@ -7,6 +7,7 @@ Commands: db-populate Populate an existing CockroachDB cluster with the Omicron schema db-wipe Wipe the Omicron schema (and all data) from an existing CockroachDB cluster ch-run Run a ClickHouse database server for development + mgs-run Run a simulated Management Gateway Service for development run-all Run a full simulated control plane cert-create Create a self-signed certificate for use with Omicron help Print this message or the help of the given subcommand(s) From 7ab9c1936a714a10e3b51ee1214cb21e08ec0af8 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 6 Oct 2023 15:36:11 +0900 Subject: [PATCH 29/89] Chore: Networking stack update (#4169) Automated dendrite updates have been stalled on: * a breaking API change in oxidecomputer/dendrite#0933cb0, * a breaking behavioural change in oxidecomputer/dendrite#616862d and its accompanying sidecar-lite/npuzone change. This PR updates these dependencies and pulls in the OPTE version needed to handle the new switch logic on ingress traffic. Once merged, Helios users will need to reinstall dependencies. --- Cargo.lock | 12 ++++++------ Cargo.toml | 4 ++-- nexus/src/app/sagas/switch_port_settings_apply.rs | 4 ++-- package-manifest.toml | 12 ++++++------ sled-agent/src/bootstrap/early_networking.rs | 3 +-- tools/ci_download_softnpu_machinery | 2 +- tools/dendrite_openapi_version | 4 ++-- tools/dendrite_stub_checksums | 6 +++--- tools/opte_version | 2 +- wicketd/src/preflight_check/uplink.rs | 2 +- 10 files changed, 25 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aad3a8782a..18e0e15c3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3390,7 +3390,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=98d33125413f01722947e322f82caf9d22209434#98d33125413f01722947e322f82caf9d22209434" +source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" [[package]] name = "illumos-utils" @@ -3811,7 +3811,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=98d33125413f01722947e322f82caf9d22209434#98d33125413f01722947e322f82caf9d22209434" +source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" dependencies = [ "quote", "syn 1.0.109", @@ -5583,7 +5583,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=98d33125413f01722947e322f82caf9d22209434#98d33125413f01722947e322f82caf9d22209434" +source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" dependencies = [ "cfg-if 0.1.10", "dyn-clone", @@ -5600,7 +5600,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=98d33125413f01722947e322f82caf9d22209434#98d33125413f01722947e322f82caf9d22209434" +source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" dependencies = [ "cfg-if 0.1.10", "illumos-sys-hdrs", @@ -5613,7 +5613,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=98d33125413f01722947e322f82caf9d22209434#98d33125413f01722947e322f82caf9d22209434" +source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" dependencies = [ "libc", "libnet", @@ -5693,7 +5693,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=98d33125413f01722947e322f82caf9d22209434#98d33125413f01722947e322f82caf9d22209434" +source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" dependencies = [ "cfg-if 0.1.10", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 1ca8a02886..3b83b2f7c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -245,7 +245,7 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.8.3" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "98d33125413f01722947e322f82caf9d22209434", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "631c2017f19cafb1535f621e9e5aa9198ccad869", features = [ "api", "std" ] } once_cell = "1.18.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "1.0" @@ -253,7 +253,7 @@ openapiv3 = "1.0" openssl = "0.10" openssl-sys = "0.9" openssl-probe = "0.1.2" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "98d33125413f01722947e322f82caf9d22209434" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "631c2017f19cafb1535f621e9e5aa9198ccad869" } oso = "0.26" owo-colors = "3.5.0" oximeter = { path = "oximeter/oximeter" } diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index 07d4dd17fb..687613f0cc 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -175,7 +175,7 @@ pub(crate) fn api_to_dpd_port_settings( .to_string(), RouteSettingsV4 { link_id: link_id.0, - nexthop: Some(gw), + nexthop: gw, vid: r.vid.map(Into::into), }, ); @@ -194,7 +194,7 @@ pub(crate) fn api_to_dpd_port_settings( .to_string(), RouteSettingsV6 { link_id: link_id.0, - nexthop: Some(gw), + nexthop: gw, vid: r.vid.map(Into::into), }, ); diff --git a/package-manifest.toml b/package-manifest.toml index ff229e5def..a7f8683eee 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -458,8 +458,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "363e365135cfa46d7f7558d8670f35aa8fe412e9" -source.sha256 = "2dc34eaac7eb9d320594f3ac125df6a601fe020e0b3c7f16eb0a5ebddc8e18b9" +source.commit = "7712104585266a2898da38c1345210ad26f9e71d" +source.sha256 = "486b0b016c0df06947810b90f3a3dd40423f0ee6f255ed079dc8e5618c9a7281" output.type = "zone" output.intermediate_only = true @@ -483,8 +483,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "363e365135cfa46d7f7558d8670f35aa8fe412e9" -source.sha256 = "1616eb25ab3d3a8b678b6cf3675af7ba61d455c3e6c2ba2a2d35a663861bc8e8" +source.commit = "7712104585266a2898da38c1345210ad26f9e71d" +source.sha256 = "76ff76d3526323c3fcbe2351cf9fbda4840e0dc11cd0eb6b71a3e0bd36c5e5e8" output.type = "zone" output.intermediate_only = true @@ -501,8 +501,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "363e365135cfa46d7f7558d8670f35aa8fe412e9" -source.sha256 = "a045e6dbb84dbceaf3a8a7dc33d283449fbeaf081442d0ae14ce8d8ffcdda4e9" +source.commit = "7712104585266a2898da38c1345210ad26f9e71d" +source.sha256 = "b8e5c176070f9bc9ea0028de1999c77d66ea3438913664163975964effe4481b" output.type = "zone" output.intermediate_only = true diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 78e54b3db4..61d4c84af3 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -495,14 +495,13 @@ impl<'a> EarlyNetworkSetup<'a> { e )) })?; - let nexthop = Some(uplink_config.gateway_ip); dpd_port_settings.v4_routes.insert( Ipv4Cidr { prefix: "0.0.0.0".parse().unwrap(), prefix_len: 0 } .to_string(), RouteSettingsV4 { link_id: link_id.0, vid: uplink_config.uplink_vid, - nexthop, + nexthop: uplink_config.gateway_ip, }, ); Ok((ipv6_entry, dpd_port_settings, port_id)) diff --git a/tools/ci_download_softnpu_machinery b/tools/ci_download_softnpu_machinery index 2575d6a186..d37d428476 100755 --- a/tools/ci_download_softnpu_machinery +++ b/tools/ci_download_softnpu_machinery @@ -15,7 +15,7 @@ OUT_DIR="out/npuzone" # Pinned commit for softnpu ASIC simulator SOFTNPU_REPO="softnpu" -SOFTNPU_COMMIT="64beaff129b7f63a04a53dd5ed0ec09f012f5756" +SOFTNPU_COMMIT="41b3a67b3d44f51528816ff8e539b4001df48305" # This is the softnpu ASIC simulator echo "fetching npuzone" diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index cbdbca7662..b1f210a647 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="363e365135cfa46d7f7558d8670f35aa8fe412e9" -SHA2="4da5edf1571a550a90aa8679a25c1535d2b02154dfb6034f170e421c2633bc31" +COMMIT="7712104585266a2898da38c1345210ad26f9e71d" +SHA2="cb3f0cfbe6216d2441d34e0470252e0fb142332e47b33b65c24ef7368a694b6d" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index acff400104..9538bc0d00 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="2dc34eaac7eb9d320594f3ac125df6a601fe020e0b3c7f16eb0a5ebddc8e18b9" -CIDL_SHA256_LINUX_DPD="5a976d1e43031f4790d1cd2f42d226b47c1be9c998917666f21cfaa3a7b13939" -CIDL_SHA256_LINUX_SWADM="38680e69364ffbfc43fea524786580d151ff45ce2f1802bd5179599f7c80e5f8" +CIDL_SHA256_ILLUMOS="486b0b016c0df06947810b90f3a3dd40423f0ee6f255ed079dc8e5618c9a7281" +CIDL_SHA256_LINUX_DPD="af97aaf7e1046a5c651d316c384171df6387b4c54c8ae4a3ef498e532eaa5a4c" +CIDL_SHA256_LINUX_SWADM="909e400dcc9880720222c6dc3919404d83687f773f668160f66f38b51a81c188" diff --git a/tools/opte_version b/tools/opte_version index 83a91f78b4..2dbaeb7154 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.23.173 +0.23.181 diff --git a/wicketd/src/preflight_check/uplink.rs b/wicketd/src/preflight_check/uplink.rs index c0f5d0c6bb..58955d04d6 100644 --- a/wicketd/src/preflight_check/uplink.rs +++ b/wicketd/src/preflight_check/uplink.rs @@ -777,7 +777,7 @@ fn build_port_settings( DPD_DEFAULT_IPV4_CIDR.parse().unwrap(), RouteSettingsV4 { link_id: link_id.0, - nexthop: Some(uplink.gateway_ip), + nexthop: uplink.gateway_ip, vid: uplink.uplink_vid, }, ); From c03029373982a528d372c68f3015506f15dd2a07 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Fri, 6 Oct 2023 07:07:06 -0700 Subject: [PATCH 30/89] Pick up correctly signed RoT sidecar images (#4216) --- tools/dvt_dock_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dvt_dock_version b/tools/dvt_dock_version index e2151b846f..790bd3ec26 100644 --- a/tools/dvt_dock_version +++ b/tools/dvt_dock_version @@ -1 +1 @@ -COMMIT=65f1979c1d3f4d0874a64144941cc41b46a70c80 +COMMIT=7cbfa19bad077a3c42976357a317d18291533ba2 From cf3bdaee3885dc34c838c5587e92787b772133a9 Mon Sep 17 00:00:00 2001 From: Patrick Mooney Date: Fri, 6 Oct 2023 16:40:00 -0500 Subject: [PATCH 31/89] Update Propolis to include UART fix This updates the Propolis packaging to use a version with the fix to oxidecomputer/propolis#540. --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 6 +++--- package-manifest.toml | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18e0e15c3b..ec4efec0fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -485,7 +485,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "bhyve_api_sys", "libc", @@ -495,7 +495,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "libc", "strum", @@ -1225,7 +1225,7 @@ dependencies = [ [[package]] name = "cpuid_profile_config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "propolis", "serde", @@ -2016,7 +2016,7 @@ checksum = "7e1a8646b2c125eeb9a84ef0faa6d2d102ea0d5da60b824ade2743263117b848" [[package]] name = "dladm" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "libc", "strum", @@ -6593,7 +6593,7 @@ dependencies = [ [[package]] name = "propolis" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "anyhow", "bhyve_api", @@ -6626,7 +6626,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "async-trait", "base64 0.21.4", @@ -6650,7 +6650,7 @@ dependencies = [ [[package]] name = "propolis-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "anyhow", "async-trait", @@ -6702,7 +6702,7 @@ dependencies = [ [[package]] name = "propolis-server-config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "cpuid_profile_config", "serde", @@ -6714,7 +6714,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "schemars", "serde", @@ -9761,7 +9761,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "viona_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "libc", "viona_api_sys", @@ -9770,7 +9770,7 @@ dependencies = [ [[package]] name = "viona_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=42c878b71a58d430dfc306126af5d40ca816d70f#42c878b71a58d430dfc306126af5d40ca816d70f" +source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" dependencies = [ "libc", ] diff --git a/Cargo.toml b/Cargo.toml index 3b83b2f7c5..9388b2c7d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -277,9 +277,9 @@ pretty-hex = "0.3.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "42c878b71a58d430dfc306126af5d40ca816d70f" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "42c878b71a58d430dfc306126af5d40ca816d70f", features = [ "generated-migration" ] } -propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "42c878b71a58d430dfc306126af5d40ca816d70f", default-features = false, features = ["mock-only"] } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "901b710b6e5bd05a94a323693c2b971e7e7b240e" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "901b710b6e5bd05a94a323693c2b971e7e7b240e", features = [ "generated-migration" ] } +propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "901b710b6e5bd05a94a323693c2b971e7e7b240e", default-features = false, features = ["mock-only"] } proptest = "1.2.0" quote = "1.0" rand = "0.8.5" diff --git a/package-manifest.toml b/package-manifest.toml index a7f8683eee..7cf235c24a 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -406,10 +406,10 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "42c878b71a58d430dfc306126af5d40ca816d70f" +source.commit = "901b710b6e5bd05a94a323693c2b971e7e7b240e" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "dce4d82bb936e990262abcaa279eee7e33a19930880b23f49fa3851cded18567" +source.sha256 = "0f681cdbe7312f66fd3c99fe033b379e49c59fa4ad04d307f68b12514307e976" output.type = "zone" [package.maghemite] From 9f004d2759b7ae827bc5e49d6cc8b5c5956573a8 Mon Sep 17 00:00:00 2001 From: Rain Date: Fri, 6 Oct 2023 21:11:29 -0700 Subject: [PATCH 32/89] [crdb-seed] use a tarball, fix omicron-dev run-all (#4208) Several changes: 1. In https://github.com/oxidecomputer/omicron/issues/4193, @david-crespo observed some missing files in the crdb-seed generated directory. My suspicion is that that is due to the `/tmp` cleaner that runs on macOS. @davepacheco suggested using a tarball to get atomicity (either the file exists or it doesn't), and it ended up being pretty simple to do that at the end. 2. Teach nexus-test-utils to ensure that the seed tarball exists, fixing `omicron-dev run-all` and anything else that uses nexus-test-utils (and avoiding a dependency on the environment). 3. Move `crdb-seed` to `dev-tools` (thanks Dave for pointing it out!) 4. Add a way to invalidate the cache with `CRDB_SEED_INVALIDATE=1` in the environment. 5. Add a readme for `crdb-seed`. Fixes #4206. Hopefully addresses #4193. --- .config/nextest.toml | 4 +- Cargo.lock | 27 +- Cargo.toml | 5 +- crdb-seed/src/main.rs | 92 ------- {crdb-seed => dev-tools/crdb-seed}/Cargo.toml | 8 +- dev-tools/crdb-seed/README.md | 11 + dev-tools/crdb-seed/src/main.rs | 39 +++ dev-tools/omicron-dev/Cargo.toml | 2 +- dev-tools/omicron-dev/src/bin/omicron-dev.rs | 10 +- .../omicron-dev/tests/test_omicron_dev.rs | 11 + nexus/test-utils/Cargo.toml | 3 + nexus/test-utils/src/db.rs | 35 ++- nexus/test-utils/src/lib.rs | 105 +++++++- test-utils/Cargo.toml | 11 +- test-utils/src/dev/mod.rs | 107 +++----- test-utils/src/dev/seed.rs | 239 ++++++++++++++++++ workspace-hack/Cargo.toml | 24 +- 17 files changed, 518 insertions(+), 215 deletions(-) delete mode 100644 crdb-seed/src/main.rs rename {crdb-seed => dev-tools/crdb-seed}/Cargo.toml (60%) create mode 100644 dev-tools/crdb-seed/README.md create mode 100644 dev-tools/crdb-seed/src/main.rs create mode 100644 test-utils/src/dev/seed.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index b2a8b360bb..ba07186c8a 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -8,7 +8,9 @@ nextest-version = { required = "0.9.59", recommended = "0.9.59" } experimental = ["setup-scripts"] [[profile.default.scripts]] -filter = 'rdeps(nexus-test-utils)' +# Exclude omicron-dev tests from crdb-seed as we explicitly want to simulate an +# environment where the seed file doesn't exist. +filter = 'rdeps(nexus-test-utils) - package(omicron-dev)' setup = 'crdb-seed' [profile.ci] diff --git a/Cargo.lock b/Cargo.lock index ec4efec0fc..306e953049 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -366,6 +366,17 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" +[[package]] +name = "atomicwrites" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1163d9d7c51de51a2b79d6df5e8888d11e9df17c752ce4a285fb6ca1580734e" +dependencies = [ + "rustix 0.37.23", + "tempfile", + "windows-sys 0.48.0", +] + [[package]] name = "atty" version = "0.2.14" @@ -1268,13 +1279,10 @@ dependencies = [ name = "crdb-seed" version = "0.1.0" dependencies = [ - "camino", - "camino-tempfile", + "anyhow", "dropshot", - "hex", "omicron-test-utils", "omicron-workspace-hack", - "ring", "slog", "tokio", ] @@ -5338,11 +5346,15 @@ name = "omicron-test-utils" version = "0.1.0" dependencies = [ "anyhow", + "atomicwrites", "camino", + "camino-tempfile", "dropshot", "expectorate", + "filetime", "futures", "headers", + "hex", "http", "libc", "omicron-common 0.1.0", @@ -5351,9 +5363,11 @@ dependencies = [ "rcgen", "regex", "reqwest", + "ring", "rustls", "slog", "subprocess", + "tar", "tempfile", "thiserror", "tokio", @@ -5436,6 +5450,7 @@ dependencies = [ "regex-syntax 0.7.5", "reqwest", "ring", + "rustix 0.37.23", "rustix 0.38.9", "schemars", "semver 1.0.18", @@ -9456,8 +9471,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", - "rand 0.4.6", + "cfg-if 1.0.0", + "rand 0.8.5", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index 9388b2c7d6..da7b582fe3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ members = [ "clients/sled-agent-client", "clients/wicketd-client", "common", - "crdb-seed", + "dev-tools/crdb-seed", "dev-tools/omdb", "dev-tools/omicron-dev", "dev-tools/thing-flinger", @@ -83,6 +83,7 @@ default-members = [ "clients/sled-agent-client", "clients/wicketd-client", "common", + "dev-tools/crdb-seed", "dev-tools/omdb", "dev-tools/omicron-dev", "dev-tools/thing-flinger", @@ -137,6 +138,7 @@ assert_matches = "1.5.0" assert_cmd = "2.0.12" async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "da04c087f835a51e0441addb19c5ef4986e1fcf2" } async-trait = "0.1.73" +atomicwrites = "0.4.1" authz-macros = { path = "nexus/authz-macros" } backoff = { version = "0.4.0", features = [ "tokio" ] } base64 = "0.21.4" @@ -182,6 +184,7 @@ dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", either = "1.9.0" expectorate = "1.1.0" fatfs = "0.3.6" +filetime = "0.2.22" flate2 = "1.0.27" flume = "0.11.0" foreign-types = "0.3.2" diff --git a/crdb-seed/src/main.rs b/crdb-seed/src/main.rs deleted file mode 100644 index b8572bd886..0000000000 --- a/crdb-seed/src/main.rs +++ /dev/null @@ -1,92 +0,0 @@ -use camino::Utf8PathBuf; -use dropshot::{test_util::LogContext, ConfigLogging, ConfigLoggingLevel}; -use omicron_test_utils::dev; -use slog::Logger; -use std::io::Write; - -// Creates a string identifier for the current DB schema and version. -// -// The goal here is to allow to create different "seed" directories -// for each revision of the DB. -fn digest_unique_to_schema() -> String { - let schema = include_str!("../../schema/crdb/dbinit.sql"); - let crdb_version = include_str!("../../tools/cockroachdb_version"); - let mut ctx = ring::digest::Context::new(&ring::digest::SHA256); - ctx.update(&schema.as_bytes()); - ctx.update(&crdb_version.as_bytes()); - let digest = ctx.finish(); - hex::encode(digest.as_ref()) -} - -enum SeedDirectoryStatus { - Created, - Existing, -} - -async fn ensure_seed_directory_exists( - log: &Logger, -) -> (Utf8PathBuf, SeedDirectoryStatus) { - let base_seed_dir = Utf8PathBuf::from_path_buf(std::env::temp_dir()) - .expect("Not a UTF-8 path") - .join("crdb-base"); - std::fs::create_dir_all(&base_seed_dir).unwrap(); - let desired_seed_dir = base_seed_dir.join(digest_unique_to_schema()); - - if desired_seed_dir.exists() { - return (desired_seed_dir, SeedDirectoryStatus::Existing); - } - - // The directory didn't exist when we started, so try to create it. - // - // Nextest will execute it just once, but it is possible for a user to start - // up multiple nextest processes to be running at the same time. So we - // should consider it possible for another caller to create this seed - // directory before we finish setting it up ourselves. - let tmp_seed_dir = - camino_tempfile::Utf8TempDir::new_in(base_seed_dir).unwrap(); - dev::test_setup_database_seed(log, tmp_seed_dir.path()).await; - - // If we can successfully perform the rename, there was either no - // contention or we won a creation race. - // - // If we couldn't perform the rename, the directory might already exist. - // Check that this is the error we encountered -- otherwise, we're - // struggling. - if let Err(err) = std::fs::rename(tmp_seed_dir.path(), &desired_seed_dir) { - if !desired_seed_dir.exists() { - panic!("Cannot rename seed directory for CockroachDB: {err}"); - } - } - - (desired_seed_dir, SeedDirectoryStatus::Created) -} - -#[tokio::main] -async fn main() { - // TODO: dropshot is v heavyweight for this, we should be able to pull in a - // smaller binary - let logctx = LogContext::new( - "crdb_seeding", - &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info }, - ); - let (dir, status) = ensure_seed_directory_exists(&logctx.log).await; - match status { - SeedDirectoryStatus::Created => { - slog::info!(logctx.log, "Created seed directory: `{dir}`"); - } - SeedDirectoryStatus::Existing => { - slog::info!(logctx.log, "Using existing seed directory: `{dir}`"); - } - } - if let Ok(env_path) = std::env::var("NEXTEST_ENV") { - let mut file = std::fs::File::create(&env_path) - .expect("failed to open NEXTEST_ENV file"); - writeln!(file, "CRDB_SEED_DIR={dir}") - .expect("failed to write to NEXTEST_ENV file"); - } else { - slog::warn!( - logctx.log, - "NEXTEST_ENV not set (is this script running under nextest?)" - ); - } -} diff --git a/crdb-seed/Cargo.toml b/dev-tools/crdb-seed/Cargo.toml similarity index 60% rename from crdb-seed/Cargo.toml rename to dev-tools/crdb-seed/Cargo.toml index 8d6d570d08..aff26995dc 100644 --- a/crdb-seed/Cargo.toml +++ b/dev-tools/crdb-seed/Cargo.toml @@ -3,14 +3,12 @@ name = "crdb-seed" version = "0.1.0" edition = "2021" license = "MPL-2.0" +readme = "README.md" [dependencies] -camino.workspace = true -camino-tempfile.workspace = true +anyhow.workspace = true dropshot.workspace = true -hex.workspace = true -omicron-test-utils.workspace = true -ring.workspace = true +omicron-test-utils = { workspace = true, features = ["seed-gen"] } slog.workspace = true tokio.workspace = true omicron-workspace-hack.workspace = true diff --git a/dev-tools/crdb-seed/README.md b/dev-tools/crdb-seed/README.md new file mode 100644 index 0000000000..3b77f23066 --- /dev/null +++ b/dev-tools/crdb-seed/README.md @@ -0,0 +1,11 @@ +# crdb-seed + +This is a small utility that creates a seed tarball for our CockroachDB instance +in the temporary directory. It is used as a setup script for nextest (see +`.config/nextest.rs`). + +This utility hashes inputs and attempts to reuse a tarball if it already exists +(see `digest_unique_to_schema` in `omicron/test-utils/src/dev/seed.rs`). + +To invalidate the tarball and cause it to be recreated from scratch, set +`CRDB_SEED_INVALIDATE=1` in the environment. diff --git a/dev-tools/crdb-seed/src/main.rs b/dev-tools/crdb-seed/src/main.rs new file mode 100644 index 0000000000..26b0e19410 --- /dev/null +++ b/dev-tools/crdb-seed/src/main.rs @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::{Context, Result}; +use dropshot::{test_util::LogContext, ConfigLogging, ConfigLoggingLevel}; +use omicron_test_utils::dev::seed::{ + ensure_seed_tarball_exists, should_invalidate_seed, +}; +use omicron_test_utils::dev::CRDB_SEED_TAR_ENV; +use std::io::Write; + +#[tokio::main] +async fn main() -> Result<()> { + // TODO: dropshot is v heavyweight for this, we should be able to pull in a + // smaller binary + let logctx = LogContext::new( + "crdb_seeding", + &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info }, + ); + let (seed_tar, status) = + ensure_seed_tarball_exists(&logctx.log, should_invalidate_seed()) + .await?; + status.log(&logctx.log, &seed_tar); + + if let Ok(env_path) = std::env::var("NEXTEST_ENV") { + let mut file = std::fs::File::create(&env_path) + .context("failed to open NEXTEST_ENV file")?; + writeln!(file, "{CRDB_SEED_TAR_ENV}={seed_tar}") + .context("failed to write to NEXTEST_ENV file")?; + } else { + slog::warn!( + logctx.log, + "NEXTEST_ENV not set (is this script running under nextest?)" + ); + } + + Ok(()) +} diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml index 251ee16c01..ec7cafb559 100644 --- a/dev-tools/omicron-dev/Cargo.toml +++ b/dev-tools/omicron-dev/Cargo.toml @@ -16,7 +16,7 @@ futures.workspace = true gateway-messages.workspace = true gateway-test-utils.workspace = true libc.workspace = true -nexus-test-utils.workspace = true +nexus-test-utils = { workspace = true, features = ["omicron-dev"] } nexus-test-interface.workspace = true omicron-common.workspace = true omicron-nexus.workspace = true diff --git a/dev-tools/omicron-dev/src/bin/omicron-dev.rs b/dev-tools/omicron-dev/src/bin/omicron-dev.rs index 9107766d8a..e79184f7e5 100644 --- a/dev-tools/omicron-dev/src/bin/omicron-dev.rs +++ b/dev-tools/omicron-dev/src/bin/omicron-dev.rs @@ -14,7 +14,6 @@ use futures::stream::StreamExt; use nexus_test_interface::NexusServer; use omicron_common::cmd::fatal; use omicron_common::cmd::CmdError; -use omicron_sled_agent::sim; use omicron_test_utils::dev; use signal_hook::consts::signal::SIGINT; use signal_hook_tokio::Signals; @@ -348,13 +347,12 @@ async fn cmd_run_all(args: &RunAllArgs) -> Result<(), anyhow::Error> { config.deployment.dropshot_external.dropshot.bind_address.set_port(p); } - // Start up a ControlPlaneTestContext, which tautologically sets up - // everything needed for a simulated control plane. println!("omicron-dev: setting up all services ... "); - let cptestctx = nexus_test_utils::test_setup_with_config::< + let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::< omicron_nexus::Server, - >("omicron-dev", &mut config, sim::SimMode::Auto, None) - .await; + >(&mut config) + .await + .context("error setting up services")?; println!("omicron-dev: services are running."); // Print out basic information about what was started. diff --git a/dev-tools/omicron-dev/tests/test_omicron_dev.rs b/dev-tools/omicron-dev/tests/test_omicron_dev.rs index f855d8935d..f1e8177243 100644 --- a/dev-tools/omicron-dev/tests/test_omicron_dev.rs +++ b/dev-tools/omicron-dev/tests/test_omicron_dev.rs @@ -13,6 +13,7 @@ use omicron_test_utils::dev::test_cmds::path_to_executable; use omicron_test_utils::dev::test_cmds::run_command; use omicron_test_utils::dev::test_cmds::EXIT_SUCCESS; use omicron_test_utils::dev::test_cmds::EXIT_USAGE; +use omicron_test_utils::dev::CRDB_SEED_TAR_ENV; use oxide_client::ClientHiddenExt; use std::io::BufRead; use std::path::Path; @@ -389,6 +390,16 @@ async fn test_db_run() { // This mirrors the `test_db_run()` test. #[tokio::test] async fn test_run_all() { + // Ensure that the CRDB_SEED_TAR environment variable is not set. We want to + // simulate a user running omicron-dev without the test environment. + // Check if CRDB_SEED_TAR_ENV is set and panic if it is + if let Ok(val) = std::env::var(CRDB_SEED_TAR_ENV) { + panic!( + "CRDB_SEED_TAR_ENV should not be set here, but is set to {}", + val + ); + } + let cmd_path = path_to_omicron_dev(); let cmdstr = format!( diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index 8eb8df4a5b..8cd25582be 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -39,3 +39,6 @@ trust-dns-proto.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true + +[features] +omicron-dev = ["omicron-test-utils/seed-gen"] diff --git a/nexus/test-utils/src/db.rs b/nexus/test-utils/src/db.rs index 37d7128c49..ff23f35df0 100644 --- a/nexus/test-utils/src/db.rs +++ b/nexus/test-utils/src/db.rs @@ -8,7 +8,7 @@ use camino::Utf8PathBuf; use omicron_test_utils::dev; use slog::Logger; -/// Path to the "seed" CockroachDB directory. +/// Path to the "seed" CockroachDB tarball. /// /// Populating CockroachDB unfortunately isn't free - creation of /// tables, indices, and users takes several seconds to complete. @@ -16,20 +16,39 @@ use slog::Logger; /// By creating a "seed" version of the database, we can cut down /// on the time spent performing this operation. Instead, we opt /// to copy the database from this seed location. -fn seed_dir() -> Utf8PathBuf { +fn seed_tar() -> Utf8PathBuf { // The setup script should set this environment variable. - let seed_dir = std::env::var("CRDB_SEED_DIR") - .expect("CRDB_SEED_DIR missing -- are you running this test with `cargo nextest run`?"); + let seed_dir = std::env::var(dev::CRDB_SEED_TAR_ENV).unwrap_or_else(|_| { + panic!( + "{} missing -- are you running this test \ + with `cargo nextest run`?", + dev::CRDB_SEED_TAR_ENV, + ) + }); seed_dir.into() } -/// Wrapper around [`dev::test_setup_database`] which uses a a -/// seed directory provided at build-time. +/// Wrapper around [`dev::test_setup_database`] which uses a seed tarball +/// provided from the environment. pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance { - let dir = seed_dir(); + let input_tar = seed_tar(); dev::test_setup_database( log, - dev::StorageSource::CopyFromSeed { input_dir: dir }, + dev::StorageSource::CopyFromSeed { input_tar }, + ) + .await +} + +/// Wrapper around [`dev::test_setup_database`] which uses a seed tarball +/// provided as an argument. +#[cfg(feature = "omicron-dev")] +pub async fn test_setup_database_from_seed( + log: &Logger, + input_tar: Utf8PathBuf, +) -> dev::db::CockroachInstance { + dev::test_setup_database( + log, + dev::StorageSource::CopyFromSeed { input_tar }, ) .await } diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index d219da7e96..34c218b3e2 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -5,6 +5,7 @@ //! Integration testing facilities for Nexus use anyhow::Context; +use anyhow::Result; use camino::Utf8Path; use dns_service_client::types::DnsConfigParams; use dropshot::test_util::ClientTestContext; @@ -284,14 +285,30 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { } pub async fn start_crdb(&mut self, populate: bool) { + let populate = if populate { + PopulateCrdb::FromEnvironmentSeed + } else { + PopulateCrdb::Empty + }; + self.start_crdb_impl(populate).await; + } + + /// Private implementation of `start_crdb` that allows for a seed tarball to + /// be passed in. See [`PopulateCrdb`] for more details. + async fn start_crdb_impl(&mut self, populate: PopulateCrdb) { let log = &self.logctx.log; debug!(log, "Starting CRDB"); // Start up CockroachDB. - let database = if populate { - db::test_setup_database(log).await - } else { - db::test_setup_database_empty(log).await + let database = match populate { + PopulateCrdb::FromEnvironmentSeed => { + db::test_setup_database(log).await + } + #[cfg(feature = "omicron-dev")] + PopulateCrdb::FromSeed { input_tar } => { + db::test_setup_database_from_seed(log, input_tar).await + } + PopulateCrdb::Empty => db::test_setup_database_empty(log).await, }; eprintln!("DB URL: {}", database.pg_config()); @@ -759,17 +776,89 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { } } +/// How to populate CockroachDB. +/// +/// This is private because we want to ensure that tests use the setup script +/// rather than trying to create their own seed tarballs. This may need to be +/// revisited if circumstances change. +#[derive(Clone, Debug)] +enum PopulateCrdb { + /// Populate Cockroach from the `CRDB_SEED_TAR_ENV` environment variable. + /// + /// Any tests that depend on nexus-test-utils should have this environment + /// variable available. + FromEnvironmentSeed, + + /// Populate Cockroach from the seed located at this path. + #[cfg(feature = "omicron-dev")] + FromSeed { input_tar: camino::Utf8PathBuf }, + + /// Do not populate Cockroach. + Empty, +} + +/// Setup routine to use for `omicron-dev`. Use [`test_setup_with_config`] for +/// tests. +/// +/// The main difference from tests is that this routine ensures the seed tarball +/// exists (or creates a seed tarball if it doesn't exist). For tests, this +/// should be done in the `crdb-seed` setup script. +#[cfg(feature = "omicron-dev")] +pub async fn omicron_dev_setup_with_config( + config: &mut omicron_common::nexus_config::Config, +) -> Result> { + let builder = + ControlPlaneTestContextBuilder::::new("omicron-dev", config); + + let log = &builder.logctx.log; + debug!(log, "Ensuring seed tarball exists"); + + // Start up a ControlPlaneTestContext, which tautologically sets up + // everything needed for a simulated control plane. + let why_invalidate = + omicron_test_utils::dev::seed::should_invalidate_seed(); + let (seed_tar, status) = + omicron_test_utils::dev::seed::ensure_seed_tarball_exists( + log, + why_invalidate, + ) + .await + .context("error ensuring seed tarball exists")?; + status.log(log, &seed_tar); + + Ok(setup_with_config_impl( + builder, + PopulateCrdb::FromSeed { input_tar: seed_tar }, + sim::SimMode::Auto, + None, + ) + .await) +} + +/// Setup routine to use for tests. pub async fn test_setup_with_config( test_name: &str, config: &mut omicron_common::nexus_config::Config, sim_mode: sim::SimMode, initial_cert: Option, ) -> ControlPlaneTestContext { - let mut builder = - ControlPlaneTestContextBuilder::::new(test_name, config); + let builder = ControlPlaneTestContextBuilder::::new(test_name, config); + setup_with_config_impl( + builder, + PopulateCrdb::FromEnvironmentSeed, + sim_mode, + initial_cert, + ) + .await +} - let populate = true; - builder.start_crdb(populate).await; +async fn setup_with_config_impl( + mut builder: ControlPlaneTestContextBuilder<'_, N>, + populate: PopulateCrdb, + sim_mode: sim::SimMode, + initial_cert: Option, +) -> ControlPlaneTestContext { + builder.start_crdb_impl(populate).await; builder.start_clickhouse().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 9e21f3ca12..7b1f70c79e 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -6,20 +6,26 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true +atomicwrites.workspace = true camino.workspace = true +camino-tempfile.workspace = true dropshot.workspace = true +filetime = { workspace = true, optional = true } futures.workspace = true headers.workspace = true +hex.workspace = true http.workspace = true libc.workspace = true omicron-common.workspace = true pem.workspace = true +ring.workspace = true rustls.workspace = true slog.workspace = true subprocess.workspace = true tempfile.workspace = true thiserror.workspace = true -tokio = { workspace = true, features = [ "full" ] } +tar.workspace = true +tokio = { workspace = true, features = ["full"] } tokio-postgres.workspace = true usdt.workspace = true rcgen.workspace = true @@ -29,3 +35,6 @@ omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true + +[features] +seed-gen = ["dep:filetime"] diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index ea95a1de76..dbd66fe1f8 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -9,55 +9,21 @@ pub mod clickhouse; pub mod db; pub mod dendrite; pub mod poll; +#[cfg(feature = "seed-gen")] +pub mod seed; pub mod test_cmds; -use anyhow::Context; -use camino::Utf8Path; +use anyhow::{Context, Result}; use camino::Utf8PathBuf; pub use dropshot::test_util::LogContext; use dropshot::ConfigLogging; use dropshot::ConfigLoggingIfExists; use dropshot::ConfigLoggingLevel; use slog::Logger; -use std::path::Path; +use std::io::BufReader; -// Helper for copying all the files in one directory to another. -fn copy_dir( - src: impl AsRef, - dst: impl AsRef, -) -> Result<(), anyhow::Error> { - let src = src.as_ref(); - let dst = dst.as_ref(); - std::fs::create_dir_all(&dst) - .with_context(|| format!("Failed to create dst {}", dst.display()))?; - for entry in std::fs::read_dir(src) - .with_context(|| format!("Failed to read_dir {}", src.display()))? - { - let entry = entry.with_context(|| { - format!("Failed to read entry in {}", src.display()) - })?; - let ty = entry.file_type().context("Failed to access file type")?; - let target = dst.join(entry.file_name()); - if ty.is_dir() { - copy_dir(entry.path(), &target).with_context(|| { - format!( - "Failed to copy subdirectory {} to {}", - entry.path().display(), - target.display() - ) - })?; - } else { - std::fs::copy(entry.path(), &target).with_context(|| { - format!( - "Failed to copy file at {} to {}", - entry.path().display(), - target.display() - ) - })?; - } - } - Ok(()) -} +/// The environment variable via which the path to the seed tarball is passed. +pub static CRDB_SEED_TAR_ENV: &str = "CRDB_SEED_TAR"; /// Set up a [`dropshot::test_util::LogContext`] appropriate for a test named /// `test_name` @@ -80,36 +46,9 @@ pub enum StorageSource { DoNotPopulate, /// Populate the latest version of the database. PopulateLatest { output_dir: Utf8PathBuf }, - /// Copy the database from a seed directory, which has previously + /// Copy the database from a seed tarball, which has previously /// been created with `PopulateLatest`. - CopyFromSeed { input_dir: Utf8PathBuf }, -} - -/// Creates a [`db::CockroachInstance`] with a populated storage directory. -/// -/// This is intended to optimize subsequent calls to [`test_setup_database`] -/// by reducing the latency of populating the storage directory. -pub async fn test_setup_database_seed(log: &Logger, dir: &Utf8Path) { - let _ = std::fs::remove_dir_all(dir); - std::fs::create_dir_all(dir).unwrap(); - let mut db = setup_database( - log, - StorageSource::PopulateLatest { output_dir: dir.to_owned() }, - ) - .await; - db.cleanup().await.unwrap(); - - // See https://github.com/cockroachdb/cockroach/issues/74231 for context on - // this. We use this assertion to check that our seed directory won't point - // back to itself, even if it is copied elsewhere. - assert_eq!( - 0, - dir.join("temp-dirs-record.txt") - .metadata() - .expect("Cannot access metadata") - .len(), - "Temporary directory record should be empty after graceful shutdown", - ); + CopyFromSeed { input_tar: Utf8PathBuf }, } /// Set up a [`db::CockroachInstance`] for running tests. @@ -118,13 +57,15 @@ pub async fn test_setup_database( source: StorageSource, ) -> db::CockroachInstance { usdt::register_probes().expect("Failed to register USDT DTrace probes"); - setup_database(log, source).await + setup_database(log, source).await.unwrap() } +// TODO: switch to anyhow entirely -- this function is currently a mishmash of +// anyhow and unwrap/expect calls. async fn setup_database( log: &Logger, storage_source: StorageSource, -) -> db::CockroachInstance { +) -> Result { let builder = db::CockroachStarterBuilder::new(); let mut builder = match &storage_source { StorageSource::DoNotPopulate | StorageSource::CopyFromSeed { .. } => { @@ -135,7 +76,7 @@ async fn setup_database( } }; builder.redirect_stdio_to_files(); - let starter = builder.build().unwrap(); + let starter = builder.build().context("error building CockroachStarter")?; info!( &log, "cockroach temporary directory: {}", @@ -147,13 +88,22 @@ async fn setup_database( match &storage_source { StorageSource::DoNotPopulate | StorageSource::PopulateLatest { .. } => { } - StorageSource::CopyFromSeed { input_dir } => { + StorageSource::CopyFromSeed { input_tar } => { info!(&log, - "cockroach: copying from seed directory ({}) to storage directory ({})", - input_dir, starter.store_dir().to_string_lossy(), + "cockroach: copying from seed tarball ({}) to storage directory ({})", + input_tar, starter.store_dir().to_string_lossy(), ); - copy_dir(input_dir, starter.store_dir()) - .expect("Cannot copy storage from seed directory"); + let reader = std::fs::File::open(input_tar).with_context(|| { + format!("cannot open input tar {}", input_tar) + })?; + let mut tar = tar::Archive::new(BufReader::new(reader)); + tar.unpack(starter.store_dir()).with_context(|| { + format!( + "cannot unpack input tar {} into {}", + input_tar, + starter.store_dir().display() + ) + })?; } } @@ -184,7 +134,8 @@ async fn setup_database( info!(&log, "cockroach: populated"); } } - database + + Ok(database) } /// Returns whether the given process is currently running diff --git a/test-utils/src/dev/seed.rs b/test-utils/src/dev/seed.rs new file mode 100644 index 0000000000..841ecd5f35 --- /dev/null +++ b/test-utils/src/dev/seed.rs @@ -0,0 +1,239 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::io::{BufWriter, Write}; + +use anyhow::{ensure, Context, Result}; +use camino::{Utf8Path, Utf8PathBuf}; +use filetime::FileTime; +use slog::Logger; + +use super::CRDB_SEED_TAR_ENV; + +/// Creates a string identifier for the current DB schema and version. +// +/// The goal here is to allow to create different "seed" tarballs +/// for each revision of the DB. +pub fn digest_unique_to_schema() -> String { + let schema = include_str!("../../../schema/crdb/dbinit.sql"); + let crdb_version = include_str!("../../../tools/cockroachdb_version"); + let mut ctx = ring::digest::Context::new(&ring::digest::SHA256); + ctx.update(&schema.as_bytes()); + ctx.update(&crdb_version.as_bytes()); + let digest = ctx.finish(); + hex::encode(digest.as_ref()) +} + +/// Looks up the standard environment variable `CRDB_SEED_INVALIDATE` to check +/// if a seed should be invalidated. Returns a string to pass in as the +/// `why_invalidate` argument of [`ensure_seed_tarball_exists`]. +pub fn should_invalidate_seed() -> Option<&'static str> { + (std::env::var("CRDB_SEED_INVALIDATE").as_deref() == Ok("1")) + .then_some("CRDB_SEED_INVALIDATE=1 set in environment") +} + +/// The return value of [`ensure_seed_tarball_exists`]. +#[derive(Clone, Copy, Debug)] +pub enum SeedTarballStatus { + Created, + Invalidated, + Existing, +} + +impl SeedTarballStatus { + pub fn log(self, log: &Logger, seed_tar: &Utf8Path) { + match self { + SeedTarballStatus::Created => { + info!(log, "Created CRDB seed tarball: `{seed_tar}`"); + } + SeedTarballStatus::Invalidated => { + info!( + log, + "Invalidated and created new CRDB seed tarball: `{seed_tar}`", + ); + } + SeedTarballStatus::Existing => { + info!(log, "Using existing CRDB seed tarball: `{seed_tar}`"); + } + } + } +} + +/// Ensures that a seed tarball corresponding to the schema returned by +/// [`digest_unique_to_schema`] exists, recreating it if necessary. +/// +/// This used to create a directory rather than a tarball, but that was changed +/// due to [Omicron issue +/// #4193](https://github.com/oxidecomputer/omicron/issues/4193). +/// +/// If `why_invalidate` is `Some`, then if the seed tarball exists, it will be +/// deleted before being recreated. +/// +/// # Notes +/// +/// This method should _not_ be used by tests. Instead, rely on the `crdb-seed` +/// setup script. +pub async fn ensure_seed_tarball_exists( + log: &Logger, + why_invalidate: Option<&str>, +) -> Result<(Utf8PathBuf, SeedTarballStatus)> { + // If the CRDB_SEED_TAR_ENV variable is set, return an error. + // + // Even though this module is gated behind a feature flag, omicron-dev needs + // this function -- and so, if you're doing a top-level `cargo nextest run` + // like CI does, feature unification would mean this gets included in test + // binaries anyway. So this acts as a belt-and-suspenders check. + if let Ok(val) = std::env::var(CRDB_SEED_TAR_ENV) { + anyhow::bail!( + "{CRDB_SEED_TAR_ENV} is set to `{val}` -- implying that a test called \ + ensure_seed_tarball_exists. Instead, tests should rely on the `crdb-seed` \ + setup script." + ); + } + + // XXX: we aren't considering cross-user permissions for this file. Might be + // worth setting more restrictive permissions on it, or using a per-user + // cache dir. + let base_seed_dir = Utf8PathBuf::from_path_buf(std::env::temp_dir()) + .expect("Not a UTF-8 path") + .join("crdb-base"); + std::fs::create_dir_all(&base_seed_dir).unwrap(); + let mut desired_seed_tar = base_seed_dir.join(digest_unique_to_schema()); + desired_seed_tar.set_extension("tar"); + + let invalidated = match (desired_seed_tar.exists(), why_invalidate) { + (true, Some(why)) => { + slog::info!( + log, + "{why}: invalidating seed tarball: `{desired_seed_tar}`", + ); + std::fs::remove_file(&desired_seed_tar) + .context("failed to remove seed tarball")?; + true + } + (true, None) => { + // The tarball exists. Update its atime and mtime (i.e. `touch` it) + // to ensure that it doesn't get deleted by a /tmp cleaner. + let now = FileTime::now(); + filetime::set_file_times(&desired_seed_tar, now, now) + .context("failed to update seed tarball atime and mtime")?; + return Ok((desired_seed_tar, SeedTarballStatus::Existing)); + } + (false, Some(why)) => { + slog::info!( + log, + "{why}, but seed tarball does not exist: `{desired_seed_tar}`", + ); + false + } + (false, None) => { + // The tarball doesn't exist. + false + } + }; + + // At this point the tarball does not exist (either because it didn't exist + // in the first place or because it was deleted above), so try to create it. + // + // Nextest will execute this function just once via the `crdb-seed` binary, + // but it is possible for a user to start up multiple nextest processes to + // be running at the same time. So we should consider it possible for + // another caller to create this seed tarball before we finish setting it up + // ourselves. + test_setup_database_seed(log, &desired_seed_tar) + .await + .context("failed to setup seed tarball")?; + + let status = if invalidated { + SeedTarballStatus::Invalidated + } else { + SeedTarballStatus::Created + }; + Ok((desired_seed_tar, status)) +} + +/// Creates a seed file for a Cockroach database at the output tarball. +/// +/// This is intended to optimize subsequent calls to +/// [`test_setup_database`](super::test_setup_database) by reducing the latency +/// of populating the storage directory. +pub async fn test_setup_database_seed( + log: &Logger, + output_tar: &Utf8Path, +) -> Result<()> { + let base_seed_dir = output_tar.parent().unwrap(); + let tmp_seed_dir = camino_tempfile::Utf8TempDir::new_in(base_seed_dir) + .context("failed to create temporary seed directory")?; + + let mut db = super::setup_database( + log, + super::StorageSource::PopulateLatest { + output_dir: tmp_seed_dir.path().to_owned(), + }, + ) + .await + .context("failed to setup database")?; + db.cleanup().await.context("failed to cleanup database")?; + + // See https://github.com/cockroachdb/cockroach/issues/74231 for context on + // this. We use this assertion to check that our seed directory won't point + // back to itself, even if it is copied elsewhere. + let dirs_record_path = tmp_seed_dir.path().join("temp-dirs-record.txt"); + let dirs_record_len = dirs_record_path + .metadata() + .with_context(|| { + format!("cannot access metadata for {dirs_record_path}") + })? + .len(); + ensure!( + dirs_record_len == 0, + "Temporary directory record should be empty (was {dirs_record_len}) \ + after graceful shutdown", + ); + + let output_tar = output_tar.to_owned(); + + tokio::task::spawn_blocking(move || { + // Tar up the directory -- this prevents issues where some but not all of + // the files get cleaned up by /tmp cleaners. See + // https://github.com/oxidecomputer/omicron/issues/4193. + let atomic_file = atomicwrites::AtomicFile::new( + &output_tar, + // We don't expect this to exist, but if it does, we want to overwrite + // it. That is because there's a remote possibility that multiple + // instances of test_setup_database_seed are running simultaneously. + atomicwrites::OverwriteBehavior::AllowOverwrite, + ); + let res = atomic_file.write(|f| { + // Tar up the directory here. + let writer = BufWriter::new(f); + let mut tar = tar::Builder::new(writer); + tar.follow_symlinks(false); + tar.append_dir_all(".", tmp_seed_dir.path()).with_context( + || { + format!( + "failed to append directory `{}` to tarball", + tmp_seed_dir.path(), + ) + }, + )?; + + let mut writer = + tar.into_inner().context("failed to finish writing tarball")?; + writer.flush().context("failed to flush tarball")?; + + Ok::<_, anyhow::Error>(()) + }); + match res { + Ok(()) => Ok(()), + Err(atomicwrites::Error::Internal(error)) => Err(error) + .with_context(|| { + format!("failed to write seed tarball: `{}`", output_tar) + }), + Err(atomicwrites::Error::User(error)) => Err(error), + } + }) + .await + .context("error in task to tar up contents")? +} diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 8854ef27bc..106da92f62 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -215,49 +215,56 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix = { version = "0.38.9", features = ["fs", "termios"] } +rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } +rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } [target.x86_64-unknown-linux-gnu.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix = { version = "0.38.9", features = ["fs", "termios"] } +rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } +rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } [target.x86_64-apple-darwin.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix = { version = "0.38.9", features = ["fs", "termios"] } +rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } +rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } [target.x86_64-apple-darwin.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix = { version = "0.38.9", features = ["fs", "termios"] } +rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } +rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } [target.aarch64-apple-darwin.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix = { version = "0.38.9", features = ["fs", "termios"] } +rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } +rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } [target.aarch64-apple-darwin.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix = { version = "0.38.9", features = ["fs", "termios"] } +rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } +rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } [target.x86_64-unknown-illumos.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix = { version = "0.38.9", features = ["fs", "termios"] } +rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } +rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } toml_edit = { version = "0.19.15", features = ["serde"] } @@ -266,7 +273,8 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix = { version = "0.38.9", features = ["fs", "termios"] } +rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } +rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } toml_edit = { version = "0.19.15", features = ["serde"] } From d624bce9af25e5bc1141de4dcdb50a15223f106d Mon Sep 17 00:00:00 2001 From: Andy Fiddaman Date: Mon, 9 Oct 2023 20:02:08 +0100 Subject: [PATCH 33/89] Back /var/fm/fmd with a dataset from the boot M.2 (#4212) `/var/fm/fmd` is where the illumos fault management system records data. We want to preserve this data across system reboots and in real time rather than via periodic data copying, so that the information is available should the system panic shortly thereafter. Fixes: https://github.com/oxidecomputer/omicron/issues/4211 --- illumos-utils/src/zfs.rs | 41 ++++-- sled-agent/src/backing_fs.rs | 178 +++++++++++++++++++++++++ sled-agent/src/bootstrap/pre_server.rs | 1 + sled-agent/src/lib.rs | 1 + sled-agent/src/sled_agent.rs | 22 ++- sled-agent/src/storage_manager.rs | 1 + sled-agent/src/swap_device.rs | 3 - sled-hardware/src/disk.rs | 15 ++- 8 files changed, 242 insertions(+), 20 deletions(-) create mode 100644 sled-agent/src/backing_fs.rs diff --git a/illumos-utils/src/zfs.rs b/illumos-utils/src/zfs.rs index ba8cd8c84a..9118a9a3cd 100644 --- a/illumos-utils/src/zfs.rs +++ b/illumos-utils/src/zfs.rs @@ -61,6 +61,9 @@ enum EnsureFilesystemErrorRaw { #[error("Failed to mount encrypted filesystem: {0}")] MountEncryptedFsFailed(crate::ExecutionError), + + #[error("Failed to mount overlay filesystem: {0}")] + MountOverlayFsFailed(crate::ExecutionError), } /// Error returned by [`Zfs::ensure_filesystem`]. @@ -202,6 +205,7 @@ impl Zfs { /// Creates a new ZFS filesystem named `name`, unless one already exists. /// /// Applies an optional quota, provided _in bytes_. + #[allow(clippy::too_many_arguments)] pub fn ensure_filesystem( name: &str, mountpoint: Mountpoint, @@ -209,6 +213,7 @@ impl Zfs { do_format: bool, encryption_details: Option, size_details: Option, + additional_options: Option>, ) -> Result<(), EnsureFilesystemError> { let (exists, mounted) = Self::dataset_exists(name, &mountpoint)?; if exists { @@ -261,7 +266,14 @@ impl Zfs { ]); } + if let Some(opts) = additional_options { + for o in &opts { + cmd.args(&["-o", &o]); + } + } + cmd.args(&["-o", &format!("mountpoint={}", mountpoint), name]); + execute(cmd).map_err(|err| EnsureFilesystemError { name: name.to_string(), mountpoint: mountpoint.clone(), @@ -322,6 +334,20 @@ impl Zfs { Ok(()) } + pub fn mount_overlay_dataset( + name: &str, + mountpoint: &Mountpoint, + ) -> Result<(), EnsureFilesystemError> { + let mut command = std::process::Command::new(PFEXEC); + let cmd = command.args(&[ZFS, "mount", "-O", name]); + execute(cmd).map_err(|err| EnsureFilesystemError { + name: name.to_string(), + mountpoint: mountpoint.clone(), + err: EnsureFilesystemErrorRaw::MountOverlayFsFailed(err), + })?; + Ok(()) + } + // Return (true, mounted) if the dataset exists, (false, false) otherwise, // where mounted is if the dataset is mounted. fn dataset_exists( @@ -385,7 +411,7 @@ impl Zfs { Zfs::get_value(filesystem_name, &format!("oxide:{}", name)) } - fn get_value( + pub fn get_value( filesystem_name: &str, name: &str, ) -> Result { @@ -422,13 +448,12 @@ pub fn get_all_omicron_datasets_for_delete() -> anyhow::Result> { let internal = pool.kind() == crate::zpool::ZpoolKind::Internal; let pool = pool.to_string(); for dataset in &Zfs::list_datasets(&pool)? { - // Avoid erasing crashdump datasets on internal pools - if dataset == "crash" && internal { - continue; - } - - // The swap device might be in use, so don't assert that it can be deleted. - if dataset == "swap" && internal { + // Avoid erasing crashdump, backing data and swap datasets on + // internal pools. The swap device may be in use. + if internal + && (["crash", "backing", "swap"].contains(&dataset.as_str()) + || dataset.starts_with("backing/")) + { continue; } diff --git a/sled-agent/src/backing_fs.rs b/sled-agent/src/backing_fs.rs new file mode 100644 index 0000000000..5014ac5999 --- /dev/null +++ b/sled-agent/src/backing_fs.rs @@ -0,0 +1,178 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Operations for dealing with persistent backing mounts for OS data + +// On Oxide hardware, the root filesystem is backed by a ramdisk and +// non-persistent. However, there are several things within the root filesystem +// which are useful to preserve across reboots, and these are backed persistent +// datasets on the boot disk. +// +// Each boot disk contains a dataset sled_hardware::disk::M2_BACKING_DATASET +// and for each backing mount, a child dataset is created under there that +// is configured with the desired mountpoint in the root filesystem. Since +// there are multiple disks which can be used to boot, these datasets are also +// marked with the "canmount=noauto" attribute so that they do not all try to +// mount automatically and race -- only one could ever succeed. This allows us +// to come along later and specifically mount the one that we want (the one from +// the current boot disk) and also perform an overlay mount so that it succeeds +// even if there is content from the ramdisk image or early boot services +// present underneath. The overlay mount action is optionally bracketed with a +// service stop/start. + +use camino::Utf8PathBuf; +use illumos_utils::zfs::{ + EnsureFilesystemError, GetValueError, Mountpoint, SizeDetails, Zfs, +}; + +#[derive(Debug, thiserror::Error)] +pub enum BackingFsError { + #[error("Error administering service: {0}")] + Adm(#[from] smf::AdmError), + + #[error("Error retrieving dataset property: {0}")] + DatasetProperty(#[from] GetValueError), + + #[error("Error initializing dataset: {0}")] + Mount(#[from] EnsureFilesystemError), +} + +struct BackingFs { + // Dataset name + name: &'static str, + // Mountpoint + mountpoint: &'static str, + // Optional quota, in _bytes_ + quota: Option, + // Optional compression mode + compression: Option<&'static str>, + // Linked service + service: Option<&'static str>, +} + +impl BackingFs { + const fn new(name: &'static str) -> Self { + Self { + name, + mountpoint: "legacy", + quota: None, + compression: None, + service: None, + } + } + + const fn mountpoint(mut self, mountpoint: &'static str) -> Self { + self.mountpoint = mountpoint; + self + } + + const fn quota(mut self, quota: usize) -> Self { + self.quota = Some(quota); + self + } + + const fn compression(mut self, compression: &'static str) -> Self { + self.compression = Some(compression); + self + } + + const fn service(mut self, service: &'static str) -> Self { + self.service = Some(service); + self + } +} + +const BACKING_FMD_DATASET: &'static str = "fmd"; +const BACKING_FMD_MOUNTPOINT: &'static str = "/var/fm/fmd"; +const BACKING_FMD_SERVICE: &'static str = "svc:/system/fmd:default"; +const BACKING_FMD_QUOTA: usize = 500 * (1 << 20); // 500 MiB + +const BACKING_COMPRESSION: &'static str = "on"; + +const BACKINGFS_COUNT: usize = 1; +static BACKINGFS: [BackingFs; BACKINGFS_COUNT] = + [BackingFs::new(BACKING_FMD_DATASET) + .mountpoint(BACKING_FMD_MOUNTPOINT) + .quota(BACKING_FMD_QUOTA) + .compression(BACKING_COMPRESSION) + .service(BACKING_FMD_SERVICE)]; + +/// Ensure that the backing filesystems are mounted. +/// If the underlying dataset for a backing fs does not exist on the specified +/// boot disk then it will be created. +pub(crate) fn ensure_backing_fs( + log: &slog::Logger, + boot_zpool_name: &illumos_utils::zpool::ZpoolName, +) -> Result<(), BackingFsError> { + let log = log.new(o!( + "component" => "BackingFs", + )); + for bfs in BACKINGFS.iter() { + info!(log, "Processing {}", bfs.name); + + let dataset = format!( + "{}/{}/{}", + boot_zpool_name, + sled_hardware::disk::M2_BACKING_DATASET, + bfs.name + ); + let mountpoint = Mountpoint::Path(Utf8PathBuf::from(bfs.mountpoint)); + + info!(log, "Ensuring dataset {}", dataset); + + let size_details = Some(SizeDetails { + quota: bfs.quota, + compression: bfs.compression, + }); + + Zfs::ensure_filesystem( + &dataset, + mountpoint.clone(), + false, // zoned + true, // do_format + None, // encryption_details, + size_details, + Some(vec!["canmount=noauto".to_string()]), // options + )?; + + // Check if a ZFS filesystem is already mounted on bfs.mountpoint by + // retrieving the ZFS `mountpoint` property and comparing it. This + // might seem counter-intuitive but if there is a filesystem mounted + // there, its mountpoint will match, and if not then we will retrieve + // the mountpoint of a higher level filesystem, such as '/'. If we + // can't retrieve the property at all, then there is definitely no ZFS + // filesystem mounted there - most likely we are running with a non-ZFS + // root, such as when net booted during CI. + if Zfs::get_value(&bfs.mountpoint, "mountpoint") + .unwrap_or("not-zfs".to_string()) + == bfs.mountpoint + { + info!(log, "{} is already mounted", bfs.mountpoint); + return Ok(()); + } + + if let Some(service) = bfs.service { + info!(log, "Stopping service {}", service); + smf::Adm::new() + .disable() + .temporary() + .synchronous() + .run(smf::AdmSelection::ByPattern(&[service]))?; + } + + info!(log, "Mounting {} on {}", dataset, mountpoint); + + Zfs::mount_overlay_dataset(&dataset, &mountpoint)?; + + if let Some(service) = bfs.service { + info!(log, "Starting service {}", service); + smf::Adm::new() + .enable() + .synchronous() + .run(smf::AdmSelection::ByPattern(&[service]))?; + } + } + + Ok(()) +} diff --git a/sled-agent/src/bootstrap/pre_server.rs b/sled-agent/src/bootstrap/pre_server.rs index 0899bdd82f..71325fef3d 100644 --- a/sled-agent/src/bootstrap/pre_server.rs +++ b/sled-agent/src/bootstrap/pre_server.rs @@ -381,6 +381,7 @@ fn ensure_zfs_ramdisk_dataset() -> Result<(), StartError> { do_format, encryption_details, quota, + None, ) .map_err(StartError::EnsureZfsRamdiskDataset) } diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index 5c4dbd8310..4e7921c605 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -17,6 +17,7 @@ pub mod sim; pub mod common; // Modules for the non-simulated sled agent. +mod backing_fs; pub mod bootstrap; pub mod config; mod http_entrypoints; diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 7e62f6a8a7..5574edca55 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -59,9 +59,15 @@ use illumos_utils::{dladm::MockDladm as Dladm, zone::MockZones as Zones}; #[derive(thiserror::Error, Debug)] pub enum Error { + #[error("Could not find boot disk")] + BootDiskNotFound, + #[error("Configuration error: {0}")] Config(#[from] crate::config::ConfigError), + #[error("Error setting up backing filesystems: {0}")] + BackingFs(#[from] crate::backing_fs::BackingFsError), + #[error("Error setting up swap device: {0}")] SwapDevice(#[from] crate::swap_device::SwapDeviceError), @@ -268,14 +274,17 @@ impl SledAgent { )); info!(&log, "SledAgent::new(..) starting"); - // Configure a swap device of the configured size before other system setup. + let boot_disk = storage + .resources() + .boot_disk() + .await + .ok_or_else(|| Error::BootDiskNotFound)?; + + // Configure a swap device of the configured size before other system + // setup. match config.swap_device_size_gb { Some(sz) if sz > 0 => { info!(log, "Requested swap device of size {} GiB", sz); - let boot_disk = - storage.resources().boot_disk().await.ok_or_else(|| { - crate::swap_device::SwapDeviceError::BootDiskNotFound - })?; crate::swap_device::ensure_swap_device( &parent_log, &boot_disk.1, @@ -290,6 +299,9 @@ impl SledAgent { } } + info!(log, "Mounting backing filesystems"); + crate::backing_fs::ensure_backing_fs(&parent_log, &boot_disk.1)?; + // Ensure we have a thread that automatically reaps process contracts // when they become empty. See the comments in // illumos-utils/src/running_zone.rs for more detail. diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index bd71371396..c31a4dc0bc 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -417,6 +417,7 @@ impl StorageWorker { do_format, encryption_details, size_details, + None, )?; // Ensure the dataset has a usable UUID. if let Ok(id_str) = Zfs::get_oxide_value(&fs_name, "uuid") { diff --git a/sled-agent/src/swap_device.rs b/sled-agent/src/swap_device.rs index 5a8f40adbd..6a00b42672 100644 --- a/sled-agent/src/swap_device.rs +++ b/sled-agent/src/swap_device.rs @@ -9,9 +9,6 @@ use zeroize::Zeroize; #[derive(Debug, thiserror::Error)] pub enum SwapDeviceError { - #[error("Could not find boot disk")] - BootDiskNotFound, - #[error("Error running ZFS command: {0}")] Zfs(illumos_utils::ExecutionError), diff --git a/sled-hardware/src/disk.rs b/sled-hardware/src/disk.rs index aec99ae3f8..e3078cbeea 100644 --- a/sled-hardware/src/disk.rs +++ b/sled-hardware/src/disk.rs @@ -256,6 +256,7 @@ pub const CRASH_DATASET: &'static str = "crash"; pub const CLUSTER_DATASET: &'static str = "cluster"; pub const CONFIG_DATASET: &'static str = "config"; pub const M2_DEBUG_DATASET: &'static str = "debug"; +pub const M2_BACKING_DATASET: &'static str = "backing"; // TODO-correctness: This value of 100GiB is a pretty wild guess, and should be // tuned as needed. pub const DEBUG_DATASET_QUOTA: usize = 100 * (1 << 30); @@ -282,7 +283,7 @@ static U2_EXPECTED_DATASETS: [ExpectedDataset; U2_EXPECTED_DATASET_COUNT] = [ .compression(DUMP_DATASET_COMPRESSION), ]; -const M2_EXPECTED_DATASET_COUNT: usize = 5; +const M2_EXPECTED_DATASET_COUNT: usize = 6; static M2_EXPECTED_DATASETS: [ExpectedDataset; M2_EXPECTED_DATASET_COUNT] = [ // Stores software images. // @@ -290,7 +291,11 @@ static M2_EXPECTED_DATASETS: [ExpectedDataset; M2_EXPECTED_DATASET_COUNT] = [ ExpectedDataset::new(INSTALL_DATASET), // Stores crash dumps. ExpectedDataset::new(CRASH_DATASET), - // Stores cluter configuration information. + // Backing store for OS data that should be persisted across reboots. + // Its children are selectively overlay mounted onto parts of the ramdisk + // root. + ExpectedDataset::new(M2_BACKING_DATASET), + // Stores cluster configuration information. // // Should be duplicated to both M.2s. ExpectedDataset::new(CLUSTER_DATASET), @@ -524,6 +529,7 @@ impl Disk { do_format, Some(encryption_details), None, + None, ); keyfile.zero_and_unlink().await.map_err(|error| { @@ -562,8 +568,8 @@ impl Disk { "Automatically destroying dataset: {}", name ); Zfs::destroy_dataset(name).or_else(|err| { - // If we can't find the dataset, that's fine -- it might - // not have been formatted yet. + // If we can't find the dataset, that's fine -- it + // might not have been formatted yet. if let DestroyDatasetErrorVariant::NotFound = err.err { @@ -588,6 +594,7 @@ impl Disk { do_format, encryption_details, size_details, + None, )?; if dataset.wipe { From 47a6b42c986c65292ee61b0c79090fa57dec5fe9 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Mon, 9 Oct 2023 16:35:01 -0500 Subject: [PATCH 34/89] [nexus] Add `/v1/ping` endpoint (#3925) Closes #3923 Adds `/v1/ping` that always returns `{ "status": "ok" }` if it returns anything at all. I went with `ping` over the initial `/v1/system/health` because the latter is vague about its meaning, whereas everyone know ping means a trivial request and response. I also thought it was weird to put an endpoint with no auth check under `/v1/system`, where ~all the other endpoints require fleet-level perms. This doesn't add too much over hitting an existing endpoint, but I think it's worth it because * It doesn't hit the DB * It has no auth check * It gives a very simple answer to "what endpoint should I use to ping the API?" (a question we have gotten at least once) * It's easy (I already did it) Questions that occurred to me while working through this: - Should we actually attempt to do something in the handler that would tell us, e.g., whether the DB is up? - No, that would be more than a ping - Raises DoS questions if not auth gated - Could add a db status endpoint or or you could use any endpoint that returns data - What tag should this be under? - Initially added a `system` tag because a) this doesn't fit under existing `system/blah` tags and b) it really does feel miscellaneous - Changed to `system/status`, with the idea that if we add other kinds of checks, they would be new endpoints under this tag. --- nexus/src/external_api/http_entrypoints.rs | 16 ++++++ nexus/src/external_api/tag-config.json | 6 ++ nexus/tests/integration_tests/basic.rs | 13 ++++- nexus/tests/integration_tests/endpoints.rs | 1 - nexus/tests/output/nexus_tags.txt | 4 ++ .../output/uncovered-authz-endpoints.txt | 1 + nexus/types/src/external_api/views.rs | 15 +++++ openapi/nexus.json | 57 +++++++++++++++++++ 8 files changed, 111 insertions(+), 2 deletions(-) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 6e614d5644..ac5cf76775 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -98,6 +98,8 @@ type NexusApiDescription = ApiDescription>; /// Returns a description of the external nexus API pub(crate) fn external_api() -> NexusApiDescription { fn register_endpoints(api: &mut NexusApiDescription) -> Result<(), String> { + api.register(ping)?; + api.register(system_policy_view)?; api.register(system_policy_update)?; @@ -364,6 +366,20 @@ pub(crate) fn external_api() -> NexusApiDescription { // clients. Client generators use operationId to name API methods, so changing // a function name is a breaking change from a client perspective. +/// Ping API +/// +/// Always responds with Ok if it responds at all. +#[endpoint { + method = GET, + path = "/v1/ping", + tags = ["system/status"], +}] +async fn ping( + _rqctx: RequestContext>, +) -> Result, HttpError> { + Ok(HttpResponseOk(views::Ping { status: views::PingStatus::Ok })) +} + /// Fetch the top-level IAM policy #[endpoint { method = GET, diff --git a/nexus/src/external_api/tag-config.json b/nexus/src/external_api/tag-config.json index e985ea7db4..07eb198016 100644 --- a/nexus/src/external_api/tag-config.json +++ b/nexus/src/external_api/tag-config.json @@ -80,6 +80,12 @@ "url": "http://docs.oxide.computer/api/vpcs" } }, + "system/status": { + "description": "Endpoints related to system health", + "external_docs": { + "url": "http://docs.oxide.computer/api/system-status" + } + }, "system/hardware": { "description": "These operations pertain to hardware inventory and management. Racks are the unit of expansion of an Oxide deployment. Racks are in turn composed of sleds, switches, power supplies, and a cabled backplane.", "external_docs": { diff --git a/nexus/tests/integration_tests/basic.rs b/nexus/tests/integration_tests/basic.rs index ab54c97197..282ec0cd96 100644 --- a/nexus/tests/integration_tests/basic.rs +++ b/nexus/tests/integration_tests/basic.rs @@ -10,7 +10,8 @@ use dropshot::HttpErrorResponseBody; use http::method::Method; use http::StatusCode; -use nexus_types::external_api::{params, views::Project}; +use nexus_types::external_api::params; +use nexus_types::external_api::views::{self, Project}; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::IdentityMetadataUpdateParams; use omicron_common::api::external::Name; @@ -546,3 +547,13 @@ async fn test_projects_list(cptestctx: &ControlPlaneTestContext) { .collect::>() ); } + +#[nexus_test] +async fn test_ping(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + + let health = NexusRequest::object_get(client, "/v1/ping") + .execute_and_parse_unwrap::() + .await; + assert_eq!(health.status, views::PingStatus::Ok); +} diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index e04d26cc45..e9ae11c21f 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -1876,6 +1876,5 @@ lazy_static! { AllowedMethod::GetNonexistent ], }, - ]; } diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index ca2f737cb0..1d7f5556c2 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -172,6 +172,10 @@ silo_view GET /v1/system/silos/{silo} user_builtin_list GET /v1/system/users-builtin user_builtin_view GET /v1/system/users-builtin/{user} +API operations found with tag "system/status" +OPERATION ID METHOD URL PATH +ping GET /v1/ping + API operations found with tag "vpcs" OPERATION ID METHOD URL PATH vpc_create POST /v1/vpcs diff --git a/nexus/tests/output/uncovered-authz-endpoints.txt b/nexus/tests/output/uncovered-authz-endpoints.txt index 0e53222a8a..d76d9c5495 100644 --- a/nexus/tests/output/uncovered-authz-endpoints.txt +++ b/nexus/tests/output/uncovered-authz-endpoints.txt @@ -1,4 +1,5 @@ API endpoints with no coverage in authz tests: +ping (get "/v1/ping") device_auth_request (post "/device/auth") device_auth_confirm (post "/device/confirm") device_access_token (post "/device/token") diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 4b30b0be1c..ef3835c618 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -522,3 +522,18 @@ pub struct UpdateDeployment { pub version: SemverVersion, pub status: UpdateStatus, } + +// SYSTEM HEALTH + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PingStatus { + Ok, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Ping { + /// Whether the external API is reachable. Will always be Ok if the endpoint + /// returns anything at all. + pub status: PingStatus, +} diff --git a/openapi/nexus.json b/openapi/nexus.json index 9330b0ef47..9dda94f283 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -2816,6 +2816,34 @@ } } }, + "/v1/ping": { + "get": { + "tags": [ + "system/status" + ], + "summary": "Ping API", + "description": "Always responds with Ok if it responds at all.", + "operationId": "ping", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Ping" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/policy": { "get": { "tags": [ @@ -12031,6 +12059,28 @@ "items" ] }, + "Ping": { + "type": "object", + "properties": { + "status": { + "description": "Whether the external API is reachable. Will always be Ok if the endpoint returns anything at all.", + "allOf": [ + { + "$ref": "#/components/schemas/PingStatus" + } + ] + } + }, + "required": [ + "status" + ] + }, + "PingStatus": { + "type": "string", + "enum": [ + "ok" + ] + }, "Project": { "description": "View of a Project", "type": "object", @@ -15277,6 +15327,13 @@ "url": "http://docs.oxide.computer/api/system-silos" } }, + { + "name": "system/status", + "description": "Endpoints related to system health", + "externalDocs": { + "url": "http://docs.oxide.computer/api/system-status" + } + }, { "name": "system/update" }, From d9d39531991cc8843ef38c4d0afc03afe1a58722 Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Tue, 10 Oct 2023 12:19:23 -0400 Subject: [PATCH 35/89] Do not double count region snapshots records! (#4095) `decrease_crucible_resource_count_and_soft_delete_volume` does not disambiguate cases where the snapshot_addr of a region_snapshot is duplicated with another one, which can occur due to the Crucible Agent reclaiming ports from destroyed daemons (see also #4049, which makes the simulated Crucible agent do this). Several invocations of the snapshot create and snapshot delete sagas could race in such a way that one of these ports would be reclaimed, and then be used in a different snapshot, and the lifetime of both of these would overlap! This would confuse our reference counting, which was written with a naive assumption that this port reuse **wouldn't** occur with these overlapping lifetimes. Spoiler alert, it can: root@[fd00:1122:3344:101::3]:32221/omicron> select * from region_snapshot where snapshot_addr = '[fd00:1122:3344:102::7]:19016'; dataset_id | region_id | snapshot_id | snapshot_addr | volume_references ---------------------------------------+--------------------------------------+--------------------------------------+-------------------------------+-------------------- 80790bfd-4b81-4381-9262-20912e3826cc | 0387bbb7-1d54-4683-943c-6c17d6804de9 | 1a800928-8f93-4cd3-9df1-4129582ffc20 | [fd00:1122:3344:102::7]:19016 | 0 80790bfd-4b81-4381-9262-20912e3826cc | ff20e066-8815-4eb6-ac84-fab9b9103462 | bdd9614e-f089-4a94-ae46-e10b96b79ba3 | [fd00:1122:3344:102::7]:19016 | 0 (2 rows) One way to solve this would be to create a UNIQUE INDEX on `snapshot_addr` here, but then in these cases the snapshot creation would return a 500 error to the user. This commit adds a sixth column: `deleting`, a boolean that is true when the region snapshot is part of a volume's `resources_to_clean_up`, and false otherwise. This is used to select (as part of the transaction for `decrease_crucible_resource_count_and_soft_delete_volume`) only the region_snapshot records that were decremented as part of that transaction, and skip re-deleting them otherwise. This works because the overlapping lifetime of the records in the DB is **not** the overlapping lifetime of the actual read-only downstairs daemon: for the port to be reclaimed, the original daemon has to be DELETEd, which happens after the decrement transaction has already computed which resources to clean up: 1) a snapshot record is created: ``` snapshot_id | snapshot_addr | volume_references | deleted | -------------------------------------+-------------------------------+-------------------+---------- 1a800928-8f93-4cd3-9df1-4129582ffc20 | [fd00:1122:3344:102::7]:19016 | 0 | false | ``` 2) it is incremented as part of `volume_create`: ``` snapshot_id | snapshot_addr | volume_references | deleted | -------------------------------------+-------------------------------+-------------------+---------- 1a800928-8f93-4cd3-9df1-4129582ffc20 | [fd00:1122:3344:102::7]:19016 | 1 | false | ``` 3) when the volume is deleted, then the decrement transaction will: a) decrease `volume_references` by 1 ``` snapshot_id | snapshot_addr | volume_references | deleted | -------------------------------------+-------------------------------+-------------------+---------- 1a800928-8f93-4cd3-9df1-4129582ffc20 | [fd00:1122:3344:102::7]:19016 | 0 | false | ``` b) note any `region_snapshot` records whose `volume_references` went to 0 and have `deleted` = false, and return those in the list of resources to clean up: [ 1a800928-8f93-4cd3-9df1-4129582ffc20 ] c) set deleted = true for any region_snapshot records whose `volume_references` went to 0 and have deleted = false ``` snapshot_id | snapshot_addr | volume_references | deleted | -------------------------------------+-------------------------------+-------------------+---------- 1a800928-8f93-4cd3-9df1-4129582ffc20 | [fd00:1122:3344:102::7]:19016 | 0 | true | ``` 4) That read-only snapshot daemon is DELETEd, freeing up the port. Another snapshot creation occurs, using that reclaimed port: ``` snapshot_id | snapshot_addr | volume_references | deleted | -------------------------------------+-------------------------------+-------------------+---------- 1a800928-8f93-4cd3-9df1-4129582ffc20 | [fd00:1122:3344:102::7]:19016 | 0 | true | bdd9614e-f089-4a94-ae46-e10b96b79ba3 | [fd00:1122:3344:102::7]:19016 | 0 | false | ``` 5) That new snapshot is incremented as part of `volume_create`: ``` snapshot_id | snapshot_addr | volume_references | deleted | -------------------------------------+-------------------------------+-------------------+---------- 1a800928-8f93-4cd3-9df1-4129582ffc20 | [fd00:1122:3344:102::7]:19016 | 0 | true | bdd9614e-f089-4a94-ae46-e10b96b79ba3 | [fd00:1122:3344:102::7]:19016 | 1 | false | ``` 6) It is later deleted, and the decrement transaction will: a) decrease `volume_references` by 1: ``` j snapshot_id | snapshot_addr | volume_references | deleted | -------------------------------------+-------------------------------+-------------------+---------- 1a800928-8f93-4cd3-9df1-4129582ffc20 | [fd00:1122:3344:102::7]:19016 | 0 | true | bdd9614e-f089-4a94-ae46-e10b96b79ba3 | [fd00:1122:3344:102::7]:19016 | 0 | false | ``` b) note any `region_snapshot` records whose `volume_references` went to 0 and have `deleted` = false, and return those in the list of resources to clean up: [ bdd9614e-f089-4a94-ae46-e10b96b79ba3 ] c) set deleted = true for any region_snapshot records whose `volume_references` went to 0 and have deleted = false ``` snapshot_id | snapshot_addr | volume_references | deleted | -------------------------------------+-------------------------------+-------------------+---------- 1a800928-8f93-4cd3-9df1-4129582ffc20 | [fd00:1122:3344:102::7]:19016 | 0 | true | bdd9614e-f089-4a94-ae46-e10b96b79ba3 | [fd00:1122:3344:102::7]:19016 | 0 | true | ``` --- dev-tools/omdb/tests/env.out | 6 +- dev-tools/omdb/tests/successes.out | 12 +- nexus/db-model/src/region_snapshot.rs | 3 + nexus/db-model/src/schema.rs | 3 +- nexus/db-queries/src/db/datastore/dataset.rs | 16 + .../src/db/datastore/region_snapshot.rs | 23 ++ nexus/db-queries/src/db/datastore/volume.rs | 100 +++--- nexus/src/app/sagas/snapshot_create.rs | 1 + nexus/src/app/sagas/volume_delete.rs | 177 ++++++---- nexus/tests/integration_tests/snapshots.rs | 36 +- .../integration_tests/volume_management.rs | 308 ++++++++++++++++++ schema/crdb/6.0.0/up1.sql | 1 + schema/crdb/6.0.0/up2.sql | 1 + schema/crdb/dbinit.sql | 5 +- 14 files changed, 563 insertions(+), 129 deletions(-) create mode 100644 schema/crdb/6.0.0/up1.sql create mode 100644 schema/crdb/6.0.0/up2.sql diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index eb4cd0d32d..07a6d3fae5 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -7,7 +7,7 @@ sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (5.0.0) +note: database schema version matches expected (6.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "--db-url", "junk", "sleds"] termination: Exited(2) @@ -172,7 +172,7 @@ stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (5.0.0) +note: database schema version matches expected (6.0.0) ============================================= EXECUTING COMMAND: omdb ["--dns-server", "[::1]:REDACTED_PORT", "db", "sleds"] termination: Exited(0) @@ -185,5 +185,5 @@ stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (5.0.0) +note: database schema version matches expected (6.0.0) ============================================= diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index eb075a84ea..038f365e8e 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -8,7 +8,7 @@ external oxide-dev.test 2 create silo: "tes --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (5.0.0) +note: database schema version matches expected (6.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "dns", "diff", "external", "2"] termination: Exited(0) @@ -24,7 +24,7 @@ changes: names added: 1, names removed: 0 --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (5.0.0) +note: database schema version matches expected (6.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "dns", "names", "external", "2"] termination: Exited(0) @@ -36,7 +36,7 @@ External zone: oxide-dev.test --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (5.0.0) +note: database schema version matches expected (6.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-instances"] termination: Exited(0) @@ -52,7 +52,7 @@ Nexus REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_ --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (5.0.0) +note: database schema version matches expected (6.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-by-sled"] termination: Exited(0) @@ -71,7 +71,7 @@ sled: sim-b6d65341 (id REDACTED_UUID_REDACTED_UUID_REDACTED) --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (5.0.0) +note: database schema version matches expected (6.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "sleds"] termination: Exited(0) @@ -82,7 +82,7 @@ sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (5.0.0) +note: database schema version matches expected (6.0.0) ============================================= EXECUTING COMMAND: omdb ["mgs", "inventory"] termination: Exited(0) diff --git a/nexus/db-model/src/region_snapshot.rs b/nexus/db-model/src/region_snapshot.rs index 9addeb83e3..af1cf8b2b3 100644 --- a/nexus/db-model/src/region_snapshot.rs +++ b/nexus/db-model/src/region_snapshot.rs @@ -32,4 +32,7 @@ pub struct RegionSnapshot { // how many volumes reference this? pub volume_references: i64, + + // true if part of a volume's `resources_to_clean_up` already + pub deleting: bool, } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 94a770e2ca..0165ab1568 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -856,6 +856,7 @@ table! { snapshot_id -> Uuid, snapshot_addr -> Text, volume_references -> Int8, + deleting -> Bool, } } @@ -1130,7 +1131,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(5, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(6, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/nexus/db-queries/src/db/datastore/dataset.rs b/nexus/db-queries/src/db/datastore/dataset.rs index 99972459c8..0b26789e8f 100644 --- a/nexus/db-queries/src/db/datastore/dataset.rs +++ b/nexus/db-queries/src/db/datastore/dataset.rs @@ -13,15 +13,31 @@ use crate::db::error::ErrorHandler; use crate::db::identity::Asset; use crate::db::model::Dataset; use crate::db::model::Zpool; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use diesel::upsert::excluded; use omicron_common::api::external::CreateResult; use omicron_common::api::external::Error; +use omicron_common::api::external::LookupResult; use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; +use uuid::Uuid; impl DataStore { + pub async fn dataset_get(&self, dataset_id: Uuid) -> LookupResult { + use db::schema::dataset::dsl; + + dsl::dataset + .filter(dsl::id.eq(dataset_id)) + .select(Dataset::as_select()) + .first_async::( + &*self.pool_connection_unauthorized().await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + /// Stores a new dataset in the database. pub async fn dataset_upsert( &self, diff --git a/nexus/db-queries/src/db/datastore/region_snapshot.rs b/nexus/db-queries/src/db/datastore/region_snapshot.rs index 0a707e4504..148cfe4812 100644 --- a/nexus/db-queries/src/db/datastore/region_snapshot.rs +++ b/nexus/db-queries/src/db/datastore/region_snapshot.rs @@ -10,9 +10,11 @@ use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::RegionSnapshot; use async_bb8_diesel::AsyncRunQueryDsl; +use async_bb8_diesel::OptionalExtension; use diesel::prelude::*; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::LookupResult; use uuid::Uuid; impl DataStore { @@ -31,6 +33,27 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + pub async fn region_snapshot_get( + &self, + dataset_id: Uuid, + region_id: Uuid, + snapshot_id: Uuid, + ) -> LookupResult> { + use db::schema::region_snapshot::dsl; + + dsl::region_snapshot + .filter(dsl::dataset_id.eq(dataset_id)) + .filter(dsl::region_id.eq(region_id)) + .filter(dsl::snapshot_id.eq(snapshot_id)) + .select(RegionSnapshot::as_select()) + .first_async::( + &*self.pool_connection_unauthorized().await?, + ) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + pub async fn region_snapshot_remove( &self, dataset_id: Uuid, diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index b3e82886de..b97b8451cf 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -119,6 +119,7 @@ impl DataStore { .filter( rs_dsl::snapshot_addr.eq(read_only_target.clone()), ) + .filter(rs_dsl::deleting.eq(false)) .set( rs_dsl::volume_references .eq(rs_dsl::volume_references + 1), @@ -573,9 +574,7 @@ impl DataStore { // multiple times, and that is done by soft-deleting the volume during // the transaction, and returning the previously serialized list of // resources to clean up if a soft-delete has already occurred. - // - // TODO it would be nice to make this transaction_async, but I couldn't - // get the async optional extension to work. + self.pool_connection_unauthorized() .await? .transaction_async(|conn| async move { @@ -639,7 +638,9 @@ impl DataStore { } }; - // Decrease the number of uses for each referenced region snapshot. + // Decrease the number of uses for each non-deleted referenced + // region snapshot. + use db::schema::region_snapshot::dsl; diesel::update(dsl::region_snapshot) @@ -647,12 +648,40 @@ impl DataStore { dsl::snapshot_addr .eq_any(crucible_targets.read_only_targets.clone()), ) + .filter(dsl::volume_references.gt(0)) + .filter(dsl::deleting.eq(false)) .set(dsl::volume_references.eq(dsl::volume_references - 1)) .execute_async(&conn) .await?; + // Then, note anything that was set to zero from the above + // UPDATE, and then mark all those as deleted. + let snapshots_to_delete: Vec = + dsl::region_snapshot + .filter( + dsl::snapshot_addr.eq_any( + crucible_targets.read_only_targets.clone(), + ), + ) + .filter(dsl::volume_references.eq(0)) + .filter(dsl::deleting.eq(false)) + .select(RegionSnapshot::as_select()) + .load_async(&conn) + .await?; + + diesel::update(dsl::region_snapshot) + .filter( + dsl::snapshot_addr + .eq_any(crucible_targets.read_only_targets.clone()), + ) + .filter(dsl::volume_references.eq(0)) + .filter(dsl::deleting.eq(false)) + .set(dsl::deleting.eq(true)) + .execute_async(&conn) + .await?; + // Return what results can be cleaned up - let result = CrucibleResources::V1(CrucibleResourcesV1 { + let result = CrucibleResources::V2(CrucibleResourcesV2 { // The only use of a read-write region will be at the top level of a // Volume. These are not shared, but if any snapshots are taken this // will prevent deletion of the region. Filter out any regions that @@ -681,6 +710,7 @@ impl DataStore { .eq(0) // Despite the SQL specifying that this column is NOT NULL, // this null check is required for this function to work! + // The left join of region_snapshot might cause a null here. .or(dsl::volume_references.is_null()), ) .select((Dataset::as_select(), Region::as_select())) @@ -688,46 +718,17 @@ impl DataStore { .await? }, - // A volume (for a disk or snapshot) may reference another nested - // volume as a read-only parent, and this may be arbitrarily deep. - // After decrementing volume_references above, get the region - // snapshot records for these read_only_targets where the - // volume_references has gone to 0. Consumers of this struct will - // be responsible for deleting the read-only downstairs running - // for the snapshot and the snapshot itself. - datasets_and_snapshots: { - use db::schema::dataset::dsl as dataset_dsl; - - dsl::region_snapshot - // Only return region_snapshot records related to - // this volume that have zero references. This will - // only happen one time, on the last decrease of a - // volume containing these read-only targets. - // - // It's important to not return *every* region - // snapshot with zero references: multiple volume - // delete sub-sagas will then be issues duplicate - // DELETE calls to Crucible agents, and a request to - // delete a read-only downstairs running for a - // snapshot that doesn't exist will return a 404, - // causing the saga to error and unwind. - .filter(dsl::snapshot_addr.eq_any( - crucible_targets.read_only_targets.clone(), - )) - .filter(dsl::volume_references.eq(0)) - .inner_join( - dataset_dsl::dataset - .on(dsl::dataset_id.eq(dataset_dsl::id)), - ) - .select(( - Dataset::as_select(), - RegionSnapshot::as_select(), - )) - .get_results_async::<(Dataset, RegionSnapshot)>( - &conn, - ) - .await? - }, + // Consumers of this struct will be responsible for deleting + // the read-only downstairs running for the snapshot and the + // snapshot itself. + // + // It's important to not return *every* region snapshot with + // zero references: multiple volume delete sub-sagas will + // then be issues duplicate DELETE calls to Crucible agents, + // and a request to delete a read-only downstairs running + // for a snapshot that doesn't exist will return a 404, + // causing the saga to error and unwind. + snapshots_to_delete, }); // Soft delete this volume, and serialize the resources that are to @@ -967,7 +968,7 @@ impl DataStore { #[derive(Default, Debug, Serialize, Deserialize)] pub struct CrucibleTargets { - read_only_targets: Vec, + pub read_only_targets: Vec, } // Serialize this enum into the `resources_to_clean_up` column to handle @@ -975,6 +976,7 @@ pub struct CrucibleTargets { #[derive(Debug, Serialize, Deserialize)] pub enum CrucibleResources { V1(CrucibleResourcesV1), + V2(CrucibleResourcesV2), } #[derive(Debug, Default, Serialize, Deserialize)] @@ -983,6 +985,12 @@ pub struct CrucibleResourcesV1 { pub datasets_and_snapshots: Vec<(Dataset, RegionSnapshot)>, } +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct CrucibleResourcesV2 { + pub datasets_and_regions: Vec<(Dataset, Region)>, + pub snapshots_to_delete: Vec, +} + /// Return the targets from a VolumeConstructionRequest. /// /// The targets of a volume construction request map to resources. diff --git a/nexus/src/app/sagas/snapshot_create.rs b/nexus/src/app/sagas/snapshot_create.rs index eeabf64894..9c8a33fb17 100644 --- a/nexus/src/app/sagas/snapshot_create.rs +++ b/nexus/src/app/sagas/snapshot_create.rs @@ -1280,6 +1280,7 @@ async fn ssc_start_running_snapshot( snapshot_id, snapshot_addr, volume_references: 0, // to be filled later + deleting: false, }) .await .map_err(ActionError::action_failed)?; diff --git a/nexus/src/app/sagas/volume_delete.rs b/nexus/src/app/sagas/volume_delete.rs index 4cd633f575..d6358d5435 100644 --- a/nexus/src/app/sagas/volume_delete.rs +++ b/nexus/src/app/sagas/volume_delete.rs @@ -155,39 +155,39 @@ async fn svd_delete_crucible_regions( sagactx.lookup::("crucible_resources_to_delete")?; // Send DELETE calls to the corresponding Crucible agents - match crucible_resources_to_delete { + let datasets_and_regions = match crucible_resources_to_delete { CrucibleResources::V1(crucible_resources_to_delete) => { - delete_crucible_regions( - log, - crucible_resources_to_delete.datasets_and_regions.clone(), - ) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "failed to delete_crucible_regions: {:?}", - e, - )) - })?; + crucible_resources_to_delete.datasets_and_regions + } - // Remove DB records - let region_ids_to_delete = crucible_resources_to_delete - .datasets_and_regions - .iter() - .map(|(_, r)| r.id()) - .collect(); - - osagactx - .datastore() - .regions_hard_delete(log, region_ids_to_delete) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "failed to regions_hard_delete: {:?}", - e, - )) - })?; + CrucibleResources::V2(crucible_resources_to_delete) => { + crucible_resources_to_delete.datasets_and_regions } - } + }; + + delete_crucible_regions(log, datasets_and_regions.clone()).await.map_err( + |e| { + ActionError::action_failed(format!( + "failed to delete_crucible_regions: {:?}", + e, + )) + }, + )?; + + // Remove DB records + let region_ids_to_delete = + datasets_and_regions.iter().map(|(_, r)| r.id()).collect(); + + osagactx + .datastore() + .regions_hard_delete(log, region_ids_to_delete) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to regions_hard_delete: {:?}", + e, + )) + })?; Ok(()) } @@ -202,26 +202,46 @@ async fn svd_delete_crucible_running_snapshots( sagactx: NexusActionContext, ) -> Result<(), ActionError> { let log = sagactx.user_data().log(); + let osagactx = sagactx.user_data(); let crucible_resources_to_delete = sagactx.lookup::("crucible_resources_to_delete")?; // Send DELETE calls to the corresponding Crucible agents - match crucible_resources_to_delete { + let datasets_and_snapshots = match crucible_resources_to_delete { CrucibleResources::V1(crucible_resources_to_delete) => { - delete_crucible_running_snapshots( - log, - crucible_resources_to_delete.datasets_and_snapshots.clone(), - ) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "failed to delete_crucible_running_snapshots: {:?}", - e, - )) - })?; + crucible_resources_to_delete.datasets_and_snapshots } - } + + CrucibleResources::V2(crucible_resources_to_delete) => { + let mut datasets_and_snapshots: Vec<_> = Vec::with_capacity( + crucible_resources_to_delete.snapshots_to_delete.len(), + ); + + for region_snapshot in + crucible_resources_to_delete.snapshots_to_delete + { + let dataset = osagactx + .datastore() + .dataset_get(region_snapshot.dataset_id) + .await + .map_err(ActionError::action_failed)?; + + datasets_and_snapshots.push((dataset, region_snapshot)); + } + + datasets_and_snapshots + } + }; + + delete_crucible_running_snapshots(log, datasets_and_snapshots.clone()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to delete_crucible_running_snapshots: {:?}", + e, + )) + })?; Ok(()) } @@ -235,26 +255,46 @@ async fn svd_delete_crucible_snapshots( sagactx: NexusActionContext, ) -> Result<(), ActionError> { let log = sagactx.user_data().log(); + let osagactx = sagactx.user_data(); let crucible_resources_to_delete = sagactx.lookup::("crucible_resources_to_delete")?; // Send DELETE calls to the corresponding Crucible agents - match crucible_resources_to_delete { + let datasets_and_snapshots = match crucible_resources_to_delete { CrucibleResources::V1(crucible_resources_to_delete) => { - delete_crucible_snapshots( - log, - crucible_resources_to_delete.datasets_and_snapshots.clone(), - ) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "failed to delete_crucible_snapshots: {:?}", - e, - )) - })?; + crucible_resources_to_delete.datasets_and_snapshots } - } + + CrucibleResources::V2(crucible_resources_to_delete) => { + let mut datasets_and_snapshots: Vec<_> = Vec::with_capacity( + crucible_resources_to_delete.snapshots_to_delete.len(), + ); + + for region_snapshot in + crucible_resources_to_delete.snapshots_to_delete + { + let dataset = osagactx + .datastore() + .dataset_get(region_snapshot.dataset_id) + .await + .map_err(ActionError::action_failed)?; + + datasets_and_snapshots.push((dataset, region_snapshot)); + } + + datasets_and_snapshots + } + }; + + delete_crucible_snapshots(log, datasets_and_snapshots.clone()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to delete_crucible_snapshots: {:?}", + e, + )) + })?; Ok(()) } @@ -293,6 +333,31 @@ async fn svd_delete_crucible_snapshot_records( })?; } } + + CrucibleResources::V2(crucible_resources_to_delete) => { + // Remove DB records + for region_snapshot in + &crucible_resources_to_delete.snapshots_to_delete + { + osagactx + .datastore() + .region_snapshot_remove( + region_snapshot.dataset_id, + region_snapshot.region_id, + region_snapshot.snapshot_id, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to region_snapshot_remove {} {} {}: {:?}", + region_snapshot.dataset_id, + region_snapshot.region_id, + region_snapshot.snapshot_id, + e, + )) + })?; + } + } } Ok(()) diff --git a/nexus/tests/integration_tests/snapshots.rs b/nexus/tests/integration_tests/snapshots.rs index d212175415..68f4cdadd2 100644 --- a/nexus/tests/integration_tests/snapshots.rs +++ b/nexus/tests/integration_tests/snapshots.rs @@ -1094,6 +1094,7 @@ async fn test_region_snapshot_create_idempotent( snapshot_addr: "[::]:12345".to_string(), volume_references: 1, + deleting: false, }; datastore.region_snapshot_create(region_snapshot.clone()).await.unwrap(); @@ -1287,13 +1288,16 @@ async fn test_multiple_deletes_not_sent(cptestctx: &ControlPlaneTestContext) { .unwrap(); let resources_1 = match resources_1 { - db::datastore::CrucibleResources::V1(resources_1) => resources_1, + db::datastore::CrucibleResources::V1(_) => panic!("using old style!"), + db::datastore::CrucibleResources::V2(resources_1) => resources_1, }; let resources_2 = match resources_2 { - db::datastore::CrucibleResources::V1(resources_2) => resources_2, + db::datastore::CrucibleResources::V1(_) => panic!("using old style!"), + db::datastore::CrucibleResources::V2(resources_2) => resources_2, }; let resources_3 = match resources_3 { - db::datastore::CrucibleResources::V1(resources_3) => resources_3, + db::datastore::CrucibleResources::V1(_) => panic!("using old style!"), + db::datastore::CrucibleResources::V2(resources_3) => resources_3, }; // No region deletions yet, these are just snapshot deletes @@ -1304,24 +1308,24 @@ async fn test_multiple_deletes_not_sent(cptestctx: &ControlPlaneTestContext) { // But there are snapshots to delete - assert!(!resources_1.datasets_and_snapshots.is_empty()); - assert!(!resources_2.datasets_and_snapshots.is_empty()); - assert!(!resources_3.datasets_and_snapshots.is_empty()); + assert!(!resources_1.snapshots_to_delete.is_empty()); + assert!(!resources_2.snapshots_to_delete.is_empty()); + assert!(!resources_3.snapshots_to_delete.is_empty()); - // Assert there are no overlaps in the datasets_and_snapshots to delete. + // Assert there are no overlaps in the snapshots_to_delete to delete. - for tuple in &resources_1.datasets_and_snapshots { - assert!(!resources_2.datasets_and_snapshots.contains(tuple)); - assert!(!resources_3.datasets_and_snapshots.contains(tuple)); + for tuple in &resources_1.snapshots_to_delete { + assert!(!resources_2.snapshots_to_delete.contains(tuple)); + assert!(!resources_3.snapshots_to_delete.contains(tuple)); } - for tuple in &resources_2.datasets_and_snapshots { - assert!(!resources_1.datasets_and_snapshots.contains(tuple)); - assert!(!resources_3.datasets_and_snapshots.contains(tuple)); + for tuple in &resources_2.snapshots_to_delete { + assert!(!resources_1.snapshots_to_delete.contains(tuple)); + assert!(!resources_3.snapshots_to_delete.contains(tuple)); } - for tuple in &resources_3.datasets_and_snapshots { - assert!(!resources_1.datasets_and_snapshots.contains(tuple)); - assert!(!resources_2.datasets_and_snapshots.contains(tuple)); + for tuple in &resources_3.snapshots_to_delete { + assert!(!resources_1.snapshots_to_delete.contains(tuple)); + assert!(!resources_2.snapshots_to_delete.contains(tuple)); } } diff --git a/nexus/tests/integration_tests/volume_management.rs b/nexus/tests/integration_tests/volume_management.rs index 70d34fb778..e263593def 100644 --- a/nexus/tests/integration_tests/volume_management.rs +++ b/nexus/tests/integration_tests/volume_management.rs @@ -19,6 +19,7 @@ use nexus_test_utils::resource_helpers::DiskTest; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; use nexus_types::external_api::views; +use nexus_types::identity::Asset; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Disk; use omicron_common::api::external::IdentityMetadataCreateParams; @@ -1813,6 +1814,313 @@ async fn test_volume_checkout_updates_sparse_mid_multiple_gen( volume_match_gen(new_vol, vec![Some(8), None, Some(10)]); } +/// Test that the Crucible agent's port reuse does not confuse +/// `decrease_crucible_resource_count_and_soft_delete_volume`, due to the +/// `[ipv6]:port` targets being reused. +#[nexus_test] +async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { + let nexus = &cptestctx.server.apictx().nexus; + let datastore = nexus.datastore(); + + // Four zpools, one dataset each + let mut disk_test = DiskTest::new(&cptestctx).await; + disk_test + .add_zpool_with_dataset(&cptestctx, DiskTest::DEFAULT_ZPOOL_SIZE_GIB) + .await; + + // This bug occurs when region_snapshot records share a snapshot_addr, so + // insert those here manually. + + // (dataset_id, region_id, snapshot_id, snapshot_addr) + let region_snapshots = vec![ + // first snapshot-create + ( + disk_test.zpools[0].datasets[0].id, + Uuid::new_v4(), + Uuid::new_v4(), + String::from("[fd00:1122:3344:101:7]:19016"), + ), + ( + disk_test.zpools[1].datasets[0].id, + Uuid::new_v4(), + Uuid::new_v4(), + String::from("[fd00:1122:3344:102:7]:19016"), + ), + ( + disk_test.zpools[2].datasets[0].id, + Uuid::new_v4(), + Uuid::new_v4(), + String::from("[fd00:1122:3344:103:7]:19016"), + ), + // second snapshot-create + ( + disk_test.zpools[0].datasets[0].id, + Uuid::new_v4(), + Uuid::new_v4(), + String::from("[fd00:1122:3344:101:7]:19016"), // duplicate! + ), + ( + disk_test.zpools[3].datasets[0].id, + Uuid::new_v4(), + Uuid::new_v4(), + String::from("[fd00:1122:3344:104:7]:19016"), + ), + ( + disk_test.zpools[2].datasets[0].id, + Uuid::new_v4(), + Uuid::new_v4(), + String::from("[fd00:1122:3344:103:7]:19017"), + ), + ]; + + // First, three `region_snapshot` records created in the snapshot-create + // saga, which are then used to make snapshot's volume construction request + + for i in 0..3 { + let (dataset_id, region_id, snapshot_id, snapshot_addr) = + ®ion_snapshots[i]; + datastore + .region_snapshot_create(nexus_db_model::RegionSnapshot { + dataset_id: *dataset_id, + region_id: *region_id, + snapshot_id: *snapshot_id, + snapshot_addr: snapshot_addr.clone(), + volume_references: 0, + deleting: false, + }) + .await + .unwrap(); + } + + let volume_id = Uuid::new_v4(); + let volume = datastore + .volume_create(nexus_db_model::Volume::new( + volume_id, + serde_json::to_string(&VolumeConstructionRequest::Volume { + id: volume_id, + block_size: 512, + sub_volumes: vec![], + read_only_parent: Some(Box::new( + VolumeConstructionRequest::Region { + block_size: 512, + blocks_per_extent: 1, + extent_count: 1, + gen: 1, + opts: CrucibleOpts { + id: Uuid::new_v4(), + target: vec![ + region_snapshots[0].3.clone(), + region_snapshots[1].3.clone(), + region_snapshots[2].3.clone(), + ], + lossy: false, + flush_timeout: None, + key: None, + cert_pem: None, + key_pem: None, + root_cert_pem: None, + control: None, + read_only: true, + }, + }, + )), + }) + .unwrap(), + )) + .await + .unwrap(); + + // Sanity check + + assert_eq!(volume.id(), volume_id); + + // Make sure the volume has only three read-only targets: + + let crucible_targets = datastore + .read_only_resources_associated_with_volume(volume_id) + .await + .unwrap(); + assert_eq!(crucible_targets.read_only_targets.len(), 3); + + // Also validate the volume's region_snapshots got incremented by + // volume_create + + for i in 0..3 { + let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let region_snapshot = datastore + .region_snapshot_get(dataset_id, region_id, snapshot_id) + .await + .unwrap() + .unwrap(); + + assert_eq!(region_snapshot.volume_references, 1); + assert_eq!(region_snapshot.deleting, false); + } + + // Soft delete the volume, and validate that only three region_snapshot + // records are returned. + + let cr = datastore + .decrease_crucible_resource_count_and_soft_delete_volume(volume_id) + .await + .unwrap(); + + for i in 0..3 { + let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let region_snapshot = datastore + .region_snapshot_get(dataset_id, region_id, snapshot_id) + .await + .unwrap() + .unwrap(); + + assert_eq!(region_snapshot.volume_references, 0); + assert_eq!(region_snapshot.deleting, true); + } + + match cr { + nexus_db_queries::db::datastore::CrucibleResources::V1(cr) => { + assert!(cr.datasets_and_regions.is_empty()); + assert_eq!(cr.datasets_and_snapshots.len(), 3); + } + + nexus_db_queries::db::datastore::CrucibleResources::V2(cr) => { + assert!(cr.datasets_and_regions.is_empty()); + assert_eq!(cr.snapshots_to_delete.len(), 3); + } + } + + // Now, let's say we're at a spot where the running snapshots have been + // deleted, but before volume_hard_delete or region_snapshot_remove are + // called. Pretend another snapshot-create and snapshot-delete snuck in + // here, and the second snapshot hits a agent that reuses the first target. + + for i in 3..6 { + let (dataset_id, region_id, snapshot_id, snapshot_addr) = + ®ion_snapshots[i]; + datastore + .region_snapshot_create(nexus_db_model::RegionSnapshot { + dataset_id: *dataset_id, + region_id: *region_id, + snapshot_id: *snapshot_id, + snapshot_addr: snapshot_addr.clone(), + volume_references: 0, + deleting: false, + }) + .await + .unwrap(); + } + + let volume_id = Uuid::new_v4(); + let volume = datastore + .volume_create(nexus_db_model::Volume::new( + volume_id, + serde_json::to_string(&VolumeConstructionRequest::Volume { + id: volume_id, + block_size: 512, + sub_volumes: vec![], + read_only_parent: Some(Box::new( + VolumeConstructionRequest::Region { + block_size: 512, + blocks_per_extent: 1, + extent_count: 1, + gen: 1, + opts: CrucibleOpts { + id: Uuid::new_v4(), + target: vec![ + region_snapshots[3].3.clone(), + region_snapshots[4].3.clone(), + region_snapshots[5].3.clone(), + ], + lossy: false, + flush_timeout: None, + key: None, + cert_pem: None, + key_pem: None, + root_cert_pem: None, + control: None, + read_only: true, + }, + }, + )), + }) + .unwrap(), + )) + .await + .unwrap(); + + // Sanity check + + assert_eq!(volume.id(), volume_id); + + // Make sure the volume has only three read-only targets: + + let crucible_targets = datastore + .read_only_resources_associated_with_volume(volume_id) + .await + .unwrap(); + assert_eq!(crucible_targets.read_only_targets.len(), 3); + + // Also validate only the volume's region_snapshots got incremented by + // volume_create. + + for i in 0..3 { + let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let region_snapshot = datastore + .region_snapshot_get(dataset_id, region_id, snapshot_id) + .await + .unwrap() + .unwrap(); + + assert_eq!(region_snapshot.volume_references, 0); + assert_eq!(region_snapshot.deleting, true); + } + for i in 3..6 { + let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let region_snapshot = datastore + .region_snapshot_get(dataset_id, region_id, snapshot_id) + .await + .unwrap() + .unwrap(); + + assert_eq!(region_snapshot.volume_references, 1); + assert_eq!(region_snapshot.deleting, false); + } + + // Soft delete the volume, and validate that only three region_snapshot + // records are returned. + + let cr = datastore + .decrease_crucible_resource_count_and_soft_delete_volume(volume_id) + .await + .unwrap(); + + // Make sure every region_snapshot is now 0, and deleting + + for i in 0..6 { + let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let region_snapshot = datastore + .region_snapshot_get(dataset_id, region_id, snapshot_id) + .await + .unwrap() + .unwrap(); + + assert_eq!(region_snapshot.volume_references, 0); + assert_eq!(region_snapshot.deleting, true); + } + + match cr { + nexus_db_queries::db::datastore::CrucibleResources::V1(cr) => { + assert!(cr.datasets_and_regions.is_empty()); + assert_eq!(cr.datasets_and_snapshots.len(), 3); + } + + nexus_db_queries::db::datastore::CrucibleResources::V2(cr) => { + assert!(cr.datasets_and_regions.is_empty()); + assert_eq!(cr.snapshots_to_delete.len(), 3); + } + } +} + #[nexus_test] async fn test_disk_create_saga_unwinds_correctly( cptestctx: &ControlPlaneTestContext, diff --git a/schema/crdb/6.0.0/up1.sql b/schema/crdb/6.0.0/up1.sql new file mode 100644 index 0000000000..4a3cdc302e --- /dev/null +++ b/schema/crdb/6.0.0/up1.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.region_snapshot ADD COLUMN IF NOT EXISTS deleting BOOL NOT NULL DEFAULT false; diff --git a/schema/crdb/6.0.0/up2.sql b/schema/crdb/6.0.0/up2.sql new file mode 100644 index 0000000000..77c136a3bf --- /dev/null +++ b/schema/crdb/6.0.0/up2.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.region_snapshot ALTER COLUMN deleting DROP DEFAULT; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index ad09092f8f..a62cbae5ea 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -505,6 +505,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.region_snapshot ( /* How many volumes reference this? */ volume_references INT8 NOT NULL, + /* Is this currently part of some resources_to_delete? */ + deleting BOOL NOT NULL, + PRIMARY KEY (dataset_id, region_id, snapshot_id) ); @@ -2574,7 +2577,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '5.0.0', NULL) + ( TRUE, NOW(), NOW(), '6.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; From 3e9f46c057b223ad390c742f882ef05e09366b77 Mon Sep 17 00:00:00 2001 From: Ryan Goodfellow Date: Tue, 10 Oct 2023 12:57:14 -0700 Subject: [PATCH 36/89] update softnpu version (#4227) This pulls in a new version of the `npuzone` tool from the softnpu repo that automatically pulls the latest sidecar-lite code. --- tools/ci_download_softnpu_machinery | 2 +- tools/create_virtual_hardware.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_download_softnpu_machinery b/tools/ci_download_softnpu_machinery index d37d428476..7975a310f0 100755 --- a/tools/ci_download_softnpu_machinery +++ b/tools/ci_download_softnpu_machinery @@ -15,7 +15,7 @@ OUT_DIR="out/npuzone" # Pinned commit for softnpu ASIC simulator SOFTNPU_REPO="softnpu" -SOFTNPU_COMMIT="41b3a67b3d44f51528816ff8e539b4001df48305" +SOFTNPU_COMMIT="eb27e6a00f1082c9faac7cf997e57d0609f7a309" # This is the softnpu ASIC simulator echo "fetching npuzone" diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index dd6d9af9dd..95c2aa63df 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -37,7 +37,7 @@ function ensure_simulated_links { dladm create-simnet -t "net$I" dladm create-simnet -t "sc${I}_0" dladm modify-simnet -t -p "net$I" "sc${I}_0" - dladm set-linkprop -p mtu=1600 "sc${I}_0" # encap headroom + dladm set-linkprop -p mtu=9000 "sc${I}_0" # match emulated devices fi success "Simnet net$I/sc${I}_0 exists" done From 97ddc7da3a5cdbded9097827f90151980755c1e4 Mon Sep 17 00:00:00 2001 From: Rain Date: Tue, 10 Oct 2023 13:12:23 -0700 Subject: [PATCH 37/89] [dependencies] add Renovate config (#4236) * Add configuration for automatically creating dependencies, and for pinning GitHub Actions digests * Add a post-upgrade script that runs cargo-hakari. Depends on https://github.com/oxidecomputer/renovate-config/pull/5. See [RFD 434](https://rfd.shared.oxide.computer/rfd/0434) and #4166. --- .github/renovate.json | 9 ++++++++ tools/renovate-post-upgrade.sh | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 .github/renovate.json create mode 100755 tools/renovate-post-upgrade.sh diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 0000000000..405a3e282b --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "local>oxidecomputer/renovate-config", + "local>oxidecomputer/renovate-config//rust/autocreate", + "local>oxidecomputer/renovate-config:post-upgrade", + "helpers:pinGitHubActionDigests" + ] +} diff --git a/tools/renovate-post-upgrade.sh b/tools/renovate-post-upgrade.sh new file mode 100755 index 0000000000..c21832e0a9 --- /dev/null +++ b/tools/renovate-post-upgrade.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# This script is run after Renovate upgrades dependencies or lock files. + +set -euo pipefail + +# Function to retry a command up to 3 times. +function retry_command { + local retries=3 + local delay=5 + local count=0 + until "$@"; do + exit_code=$? + count=$((count+1)) + if [ $count -lt $retries ]; then + echo "Command failed with exit code $exit_code. Retrying in $delay seconds..." + sleep $delay + else + echo "Command failed with exit code $exit_code after $count attempts." + return $exit_code + fi + done +} + +# Download and install cargo-hakari if it is not already installed. +if ! command -v cargo-hakari &> /dev/null; then + # Need cargo-binstall to install cargo-hakari. + if ! command -v cargo-binstall &> /dev/null; then + # Fetch cargo binstall. + echo "Installing cargo-binstall..." + curl --retry 3 -L --proto '=https' --tlsv1.2 -sSfO https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh + retry_command bash install-from-binstall-release.sh + fi + + # Install cargo-hakari. + echo "Installing cargo-hakari..." + retry_command cargo binstall cargo-hakari --no-confirm +fi + +# Run cargo hakari to regenerate the workspace-hack file. +echo "Running cargo-hakari..." +cargo hakari generate From 72a0429debfaf4feeec2f952fefe3ffbffeb06f6 Mon Sep 17 00:00:00 2001 From: Rain Date: Tue, 10 Oct 2023 17:50:06 -0700 Subject: [PATCH 38/89] [update-engine] fix buffer tests (#4163) Apparently I'd made a couple of mistakes while writing tests: * I was adding all events a second time by accident, which was hiding the fact that... * A couple not signs were flipped, whoops. --- update-engine/src/buffer.rs | 46 ++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/update-engine/src/buffer.rs b/update-engine/src/buffer.rs index 3de0e45f24..1779ef7da6 100644 --- a/update-engine/src/buffer.rs +++ b/update-engine/src/buffer.rs @@ -1389,7 +1389,10 @@ mod tests { test_cx .run_filtered_test( "all events passed in", - |buffer, event| buffer.add_event(event.clone()), + |buffer, event| { + buffer.add_event(event.clone()); + true + }, WithDeltas::No, ) .unwrap(); @@ -1397,10 +1400,12 @@ mod tests { test_cx .run_filtered_test( "progress events skipped", - |buffer, event| { - if let Event::Step(event) = event { + |buffer, event| match event { + Event::Step(event) => { buffer.add_step_event(event.clone()); + true } + Event::Progress(_) => false, }, WithDeltas::Both, ) @@ -1410,13 +1415,16 @@ mod tests { .run_filtered_test( "low-priority events skipped", |buffer, event| match event { - Event::Step(event) => { - if event.kind.priority() == StepEventPriority::Low { + Event::Step(event) => match event.kind.priority() { + StepEventPriority::High => { buffer.add_step_event(event.clone()); + true } - } + StepEventPriority::Low => false, + }, Event::Progress(event) => { buffer.add_progress_event(event.clone()); + true } }, WithDeltas::Both, @@ -1427,13 +1435,16 @@ mod tests { .run_filtered_test( "low-priority and progress events skipped", |buffer, event| match event { - Event::Step(event) => { - if event.kind.priority() == StepEventPriority::Low { + Event::Step(event) => match event.kind.priority() { + StepEventPriority::High => { buffer.add_step_event(event.clone()); + true } - } + StepEventPriority::Low => false, + }, Event::Progress(_) => { - // Don't add progress events either. + // Don't add progress events. + false } }, WithDeltas::Both, @@ -1565,7 +1576,10 @@ mod tests { fn run_filtered_test( &self, event_fn_description: &str, - mut event_fn: impl FnMut(&mut EventBuffer, &Event), + mut event_fn: impl FnMut( + &mut EventBuffer, + &Event, + ) -> bool, with_deltas: WithDeltas, ) -> anyhow::Result<()> { match with_deltas { @@ -1590,7 +1604,10 @@ mod tests { fn run_filtered_test_inner( &self, - mut event_fn: impl FnMut(&mut EventBuffer, &Event), + mut event_fn: impl FnMut( + &mut EventBuffer, + &Event, + ) -> bool, with_deltas: bool, ) -> anyhow::Result<()> { let description = format!("with deltas = {with_deltas}"); @@ -1608,8 +1625,9 @@ mod tests { let mut last_seen_opt = with_deltas.then_some(None); for (i, event) in self.generated_events.iter().enumerate() { - (event_fn)(&mut buffer, event); - buffer.add_event(event.clone()); + // Going to use event_added in an upcoming commit. + let _event_added = (event_fn)(&mut buffer, event); + let report = match last_seen_opt { Some(last_seen) => buffer.generate_report_since(last_seen), None => buffer.generate_report(), From 194889b956abbb3e01ce25b11b733c02598c3215 Mon Sep 17 00:00:00 2001 From: Rain Date: Tue, 10 Oct 2023 19:27:33 -0700 Subject: [PATCH 39/89] [buildomat] authorize PRs generated by oxide-renovate (#4244) Means that PRs like https://github.com/oxidecomputer/omicron/pull/4241 will be automatically authorized. Also skip cargo-hakari update if cargo isn't present. --- .github/buildomat/config.toml | 1 + tools/renovate-post-upgrade.sh | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.github/buildomat/config.toml b/.github/buildomat/config.toml index 922de631f2..419173fa50 100644 --- a/.github/buildomat/config.toml +++ b/.github/buildomat/config.toml @@ -17,5 +17,6 @@ org_only = true allow_users = [ "dependabot[bot]", "oxide-reflector-bot[bot]", + "oxide-renovate[bot]", "renovate[bot]", ] diff --git a/tools/renovate-post-upgrade.sh b/tools/renovate-post-upgrade.sh index c21832e0a9..2699f9f6a0 100755 --- a/tools/renovate-post-upgrade.sh +++ b/tools/renovate-post-upgrade.sh @@ -22,6 +22,13 @@ function retry_command { done } +# If cargo isn't present, skip this -- it implies that a non-Rust dependency was +# updated. +if ! command -v cargo &> /dev/null; then + echo "Skipping cargo-hakari update because cargo is not present." + exit 0 +fi + # Download and install cargo-hakari if it is not already installed. if ! command -v cargo-hakari &> /dev/null; then # Need cargo-binstall to install cargo-hakari. From a972c80c407b68848c178aca236ae00067bc4d3b Mon Sep 17 00:00:00 2001 From: Andy Fiddaman Date: Wed, 11 Oct 2023 17:59:57 +0100 Subject: [PATCH 40/89] destroy_virtual_hardware.sh needs to unmount backing filesystems (#4255) The backing filesystems added in d624bce9af2 prevent the destroy_virtual_hardware.sh script from properly cleaning up all ZFS pools and cause the fmd service to go into maintenance which delays control plane startup. This updates the script to unwind the backing datasets as part of its work. --- tools/destroy_virtual_hardware.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools/destroy_virtual_hardware.sh b/tools/destroy_virtual_hardware.sh index ae6fef0673..46c6f117c4 100755 --- a/tools/destroy_virtual_hardware.sh +++ b/tools/destroy_virtual_hardware.sh @@ -56,7 +56,23 @@ function remove_softnpu_zone { --ports sc0_1,tfportqsfp0_0 } +# Some services have their working data overlaid by backing mounts from the +# internal boot disk. Before we can destroy the ZFS pools, we need to unmount +# these. + +BACKED_SERVICES="svc:/system/fmd:default" + +function demount_backingfs { + svcadm disable -st $BACKED_SERVICES + zpool list -Hpo name | grep '^oxi_' \ + | xargs -i zfs list -Hpo name,canmount,mounted -r {}/backing \ + | awk '$3 == "yes" && $2 == "noauto" { print $1 }' \ + | xargs -l zfs umount + svcadm enable -st $BACKED_SERVICES +} + verify_omicron_uninstalled +demount_backingfs unload_xde_driver remove_softnpu_zone try_remove_vnics From 1a21fdd581d80d92287c9f29e095dbee11f65b28 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 12:23:46 -0700 Subject: [PATCH 41/89] Update Rust crate proptest to 1.3.1 (#4243) Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com> --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 306e953049..421bbd5e16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6737,19 +6737,19 @@ dependencies = [ [[package]] name = "proptest" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e35c06b98bf36aba164cc17cb25f7e232f5c4aeea73baa14b8a9f0d92dbfa65" +checksum = "7c003ac8c77cb07bb74f5f198bce836a689bcd5a42574612bf14d17bfd08c20e" dependencies = [ "bit-set", - "bitflags 1.3.2", - "byteorder", + "bit-vec", + "bitflags 2.4.0", "lazy_static", "num-traits", "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.6.29", + "regex-syntax 0.7.5", "rusty-fork", "tempfile", "unarray", diff --git a/Cargo.toml b/Cargo.toml index da7b582fe3..fdd67c3b5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -283,7 +283,7 @@ progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branc bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "901b710b6e5bd05a94a323693c2b971e7e7b240e" } propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "901b710b6e5bd05a94a323693c2b971e7e7b240e", features = [ "generated-migration" ] } propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "901b710b6e5bd05a94a323693c2b971e7e7b240e", default-features = false, features = ["mock-only"] } -proptest = "1.2.0" +proptest = "1.3.1" quote = "1.0" rand = "0.8.5" ratatui = "0.23.0" From 02aef4bec751b47b6d19adbeef9e51c42c10204d Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 12:34:34 -0700 Subject: [PATCH 42/89] Update Rust crate predicates to 3.0.4 (#4254) Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- workspace-hack/Cargo.toml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 421bbd5e16..d58ba77133 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -283,7 +283,7 @@ dependencies = [ "anstyle", "bstr 1.6.0", "doc-comment", - "predicates 3.0.3", + "predicates 3.0.4", "predicates-core", "predicates-tree", "wait-timeout", @@ -5442,7 +5442,7 @@ dependencies = [ "phf_shared 0.11.2", "postgres-types", "ppv-lite86", - "predicates 3.0.3", + "predicates 3.0.4", "rand 0.8.5", "rand_chacha 0.3.1", "regex", @@ -6437,14 +6437,14 @@ dependencies = [ [[package]] name = "predicates" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9" +checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" dependencies = [ "anstyle", "difflib", "float-cmp", - "itertools 0.10.5", + "itertools 0.11.0", "normalize-line-endings", "predicates-core", "regex", @@ -9374,7 +9374,7 @@ dependencies = [ "omicron-common 0.1.0", "omicron-test-utils", "omicron-workspace-hack", - "predicates 3.0.3", + "predicates 3.0.4", "slog", "slog-async", "slog-envlogger", diff --git a/Cargo.toml b/Cargo.toml index fdd67c3b5c..832b8663e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -274,7 +274,7 @@ percent-encoding = "2.2.0" pem = "1.1" petgraph = "0.6.4" postgres-protocol = "0.6.6" -predicates = "3.0.3" +predicates = "3.0.4" pretty_assertions = "1.4.0" pretty-hex = "0.3.0" proc-macro2 = "1.0" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 106da92f62..a91477678b 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -73,7 +73,7 @@ petgraph = { version = "0.6.4", features = ["serde-1"] } phf_shared = { version = "0.11.2" } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } -predicates = { version = "3.0.3" } +predicates = { version = "3.0.4" } rand = { version = "0.8.5", features = ["min_const_gen", "small_rng"] } rand_chacha = { version = "0.3.1" } regex = { version = "1.9.5" } @@ -171,7 +171,7 @@ petgraph = { version = "0.6.4", features = ["serde-1"] } phf_shared = { version = "0.11.2" } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } -predicates = { version = "3.0.3" } +predicates = { version = "3.0.4" } rand = { version = "0.8.5", features = ["min_const_gen", "small_rng"] } rand_chacha = { version = "0.3.1" } regex = { version = "1.9.5" } From d12cb0ffceeb09c1cccdada29ca24c3829a2c9fa Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 12:37:42 -0700 Subject: [PATCH 43/89] Pin GitHub Actions dependencies (#4240) --- .github/workflows/check-opte-ver.yml | 2 +- .github/workflows/check-workspace-deps.yml | 2 +- .github/workflows/hakari.yml | 10 +++++----- .github/workflows/rust.yml | 14 +++++++------- .github/workflows/update-dendrite.yml | 2 +- .github/workflows/update-maghemite.yml | 2 +- .github/workflows/validate-openapi-spec.yml | 4 ++-- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/check-opte-ver.yml b/.github/workflows/check-opte-ver.yml index 3b57f2795f..a8e18f080e 100644 --- a/.github/workflows/check-opte-ver.yml +++ b/.github/workflows/check-opte-ver.yml @@ -9,7 +9,7 @@ jobs: check-opte-ver: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 - name: Install jq run: sudo apt-get install -y jq - name: Install toml-cli diff --git a/.github/workflows/check-workspace-deps.yml b/.github/workflows/check-workspace-deps.yml index 9611c4103c..521afa7359 100644 --- a/.github/workflows/check-workspace-deps.yml +++ b/.github/workflows/check-workspace-deps.yml @@ -10,6 +10,6 @@ jobs: check-workspace-deps: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 - name: Check Workspace Dependencies run: cargo xtask check-workspace-deps diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index d79196d318..df4cbc9b59 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -17,21 +17,21 @@ jobs: env: RUSTFLAGS: -D warnings steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@e659bf85ee986e37e35cc1c53bfeebe044d8133e # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date - uses: actions-rs/cargo@v1 + uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 with: command: hakari args: generate --diff - name: Check all crates depend on workspace-hack - uses: actions-rs/cargo@v1 + uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 with: command: hakari args: manage-deps --dry-run diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f5cf1dc885..873b316e16 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -9,7 +9,7 @@ jobs: check-style: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 - name: Report cargo version run: cargo --version - name: Report rustfmt version @@ -27,8 +27,8 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@v3.5.0 - - uses: Swatinem/rust-cache@v2.2.1 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: Swatinem/rust-cache@6fd3edff6979b79f87531400ad694fb7f2c84b1f # v2.2.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version @@ -53,8 +53,8 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@v3.5.0 - - uses: Swatinem/rust-cache@v2.2.1 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: Swatinem/rust-cache@6fd3edff6979b79f87531400ad694fb7f2c84b1f # v2.2.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version @@ -79,8 +79,8 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@v3.5.0 - - uses: Swatinem/rust-cache@v2.2.1 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: Swatinem/rust-cache@6fd3edff6979b79f87531400ad694fb7f2c84b1f # v2.2.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version diff --git a/.github/workflows/update-dendrite.yml b/.github/workflows/update-dendrite.yml index 86049dcafc..10d8ef7618 100644 --- a/.github/workflows/update-dendrite.yml +++ b/.github/workflows/update-dendrite.yml @@ -29,7 +29,7 @@ jobs: steps: # Checkout both the target and integration branches - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 with: token: ${{ inputs.reflector_access_token }} fetch-depth: 0 diff --git a/.github/workflows/update-maghemite.yml b/.github/workflows/update-maghemite.yml index 07fe329af3..7aa2b8b6c8 100644 --- a/.github/workflows/update-maghemite.yml +++ b/.github/workflows/update-maghemite.yml @@ -29,7 +29,7 @@ jobs: steps: # Checkout both the target and integration branches - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 with: token: ${{ inputs.reflector_access_token }} fetch-depth: 0 diff --git a/.github/workflows/validate-openapi-spec.yml b/.github/workflows/validate-openapi-spec.yml index 06fc7526a8..1d6c152296 100644 --- a/.github/workflows/validate-openapi-spec.yml +++ b/.github/workflows/validate-openapi-spec.yml @@ -10,8 +10,8 @@ jobs: format: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3.5.0 - - uses: actions/setup-node@v3.6.0 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 with: node-version: '18' - name: Install our tools From 7d335441ad87b17e7ff1bea3ea04b16d47e5567e Mon Sep 17 00:00:00 2001 From: Rain Date: Wed, 11 Oct 2023 15:29:43 -0700 Subject: [PATCH 44/89] [renovate] download install-from-binstall-release.sh into a temp dir (#4260) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whoops, this would leave an untracked file in the repo if run (and then accidentally be checked in 😬) --- tools/renovate-post-upgrade.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/renovate-post-upgrade.sh b/tools/renovate-post-upgrade.sh index 2699f9f6a0..4a9e3aa2f2 100755 --- a/tools/renovate-post-upgrade.sh +++ b/tools/renovate-post-upgrade.sh @@ -35,8 +35,10 @@ if ! command -v cargo-hakari &> /dev/null; then if ! command -v cargo-binstall &> /dev/null; then # Fetch cargo binstall. echo "Installing cargo-binstall..." - curl --retry 3 -L --proto '=https' --tlsv1.2 -sSfO https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh - retry_command bash install-from-binstall-release.sh + tempdir=$(mktemp -d) + curl --retry 3 -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh -o "$tempdir"/install-from-binstall-release.sh + retry_command bash "$tempdir"/install-from-binstall-release.sh + rm -rf "$tempdir" fi # Install cargo-hakari. From a903d61bcc8813f0ef4fbe974f469ff4619e1cc0 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 11 Oct 2023 18:31:55 -0700 Subject: [PATCH 45/89] Simplify Diesel Error management (#4210) Depends on https://github.com/oxidecomputer/async-bb8-diesel/pull/54 As of https://github.com/oxidecomputer/omicron/pull/4140 , we check out connections before issuing queries to the underlying database. This means that when we receive errors from the database, they are not overloaded as "connection checkout" OR "database" errors - they are now always database errors. --- Cargo.lock | 2 +- Cargo.toml | 2 +- nexus/db-queries/src/db/collection_attach.rs | 10 +- nexus/db-queries/src/db/collection_detach.rs | 5 +- .../src/db/collection_detach_many.rs | 8 +- nexus/db-queries/src/db/collection_insert.rs | 18 ++- .../src/db/datastore/address_lot.rs | 29 ++--- .../src/db/datastore/db_metadata.rs | 2 +- .../src/db/datastore/device_auth.rs | 2 +- nexus/db-queries/src/db/datastore/dns.rs | 2 +- .../src/db/datastore/external_ip.rs | 3 +- nexus/db-queries/src/db/datastore/ip_pool.rs | 64 +++++------ nexus/db-queries/src/db/datastore/mod.rs | 13 +-- .../src/db/datastore/network_interface.rs | 3 +- nexus/db-queries/src/db/datastore/project.rs | 24 ++-- nexus/db-queries/src/db/datastore/rack.rs | 7 +- .../src/db/datastore/region_snapshot.rs | 2 +- nexus/db-queries/src/db/datastore/role.rs | 2 +- nexus/db-queries/src/db/datastore/silo.rs | 24 ++-- .../db-queries/src/db/datastore/silo_group.rs | 5 +- nexus/db-queries/src/db/datastore/sled.rs | 2 +- nexus/db-queries/src/db/datastore/snapshot.rs | 16 +-- .../src/db/datastore/switch_interface.rs | 24 ++-- .../src/db/datastore/switch_port.rs | 104 +++++++----------- nexus/db-queries/src/db/datastore/update.rs | 2 +- nexus/db-queries/src/db/datastore/volume.rs | 8 +- nexus/db-queries/src/db/datastore/vpc.rs | 49 ++++----- nexus/db-queries/src/db/error.rs | 79 ++++--------- nexus/db-queries/src/db/explain.rs | 7 +- nexus/db-queries/src/db/pool.rs | 4 +- .../db-queries/src/db/queries/external_ip.rs | 3 +- .../src/db/queries/network_interface.rs | 96 +++++++--------- .../src/db/queries/region_allocation.rs | 3 +- nexus/db-queries/src/db/queries/vpc_subnet.rs | 20 ++-- nexus/db-queries/src/db/true_or_cast_error.rs | 9 +- nexus/db-queries/src/db/update_and_check.rs | 3 +- nexus/src/app/sagas/disk_create.rs | 5 +- nexus/src/app/sagas/instance_create.rs | 4 +- nexus/src/app/sagas/project_create.rs | 5 +- nexus/src/app/sagas/snapshot_create.rs | 6 +- nexus/src/app/sagas/vpc_create.rs | 6 +- 41 files changed, 291 insertions(+), 391 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d58ba77133..d5a90f7f85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -298,7 +298,7 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-bb8-diesel" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=da04c087f835a51e0441addb19c5ef4986e1fcf2#da04c087f835a51e0441addb19c5ef4986e1fcf2" +source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=1446f7e0c1f05f33a0581abd51fa873c7652ab61#1446f7e0c1f05f33a0581abd51fa873c7652ab61" dependencies = [ "async-trait", "bb8", diff --git a/Cargo.toml b/Cargo.toml index 832b8663e6..7521bb4d45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,7 +136,7 @@ api_identity = { path = "api_identity" } approx = "0.5.1" assert_matches = "1.5.0" assert_cmd = "2.0.12" -async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "da04c087f835a51e0441addb19c5ef4986e1fcf2" } +async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "1446f7e0c1f05f33a0581abd51fa873c7652ab61" } async-trait = "0.1.73" atomicwrites = "0.4.1" authz-macros = { path = "nexus/authz-macros" } diff --git a/nexus/db-queries/src/db/collection_attach.rs b/nexus/db-queries/src/db/collection_attach.rs index 40ec659bf9..ea4d9d5beb 100644 --- a/nexus/db-queries/src/db/collection_attach.rs +++ b/nexus/db-queries/src/db/collection_attach.rs @@ -17,7 +17,7 @@ use super::cte_utils::{ QueryFromClause, QuerySqlType, TableDefaultWhereClause, }; use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionError}; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::associations::HasTable; use diesel::expression::{AsExpression, Expression}; use diesel::helper_types::*; @@ -26,6 +26,7 @@ use diesel::prelude::*; use diesel::query_builder::*; use diesel::query_dsl::methods as query_methods; use diesel::query_source::Table; +use diesel::result::Error as DieselError; use diesel::sql_types::{BigInt, Nullable, SingleValue}; use nexus_db_model::DatastoreAttachTargetConfig; use std::fmt::Debug; @@ -299,7 +300,7 @@ where /// Result of [`AttachToCollectionStatement`] when executed asynchronously pub type AsyncAttachToCollectionResult = - Result<(C, ResourceType), AttachError>; + Result<(C, ResourceType), AttachError>; /// Errors returned by [`AttachToCollectionStatement`]. #[derive(Debug)] @@ -998,9 +999,8 @@ mod test { .set(resource::dsl::collection_id.eq(collection_id)), ); - type TxnError = TransactionError< - AttachError, - >; + type TxnError = + TransactionError>; let result = conn .transaction_async(|conn| async move { attach_query.attach_and_get_result_async(&conn).await.map_err( diff --git a/nexus/db-queries/src/db/collection_detach.rs b/nexus/db-queries/src/db/collection_detach.rs index df157040e6..03e09d41ca 100644 --- a/nexus/db-queries/src/db/collection_detach.rs +++ b/nexus/db-queries/src/db/collection_detach.rs @@ -16,7 +16,7 @@ use super::cte_utils::{ QueryFromClause, QuerySqlType, }; use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionError}; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::associations::HasTable; use diesel::expression::{AsExpression, Expression}; use diesel::helper_types::*; @@ -25,6 +25,7 @@ use diesel::prelude::*; use diesel::query_builder::*; use diesel::query_dsl::methods as query_methods; use diesel::query_source::Table; +use diesel::result::Error as DieselError; use diesel::sql_types::{Nullable, SingleValue}; use nexus_db_model::DatastoreAttachTargetConfig; use std::fmt::Debug; @@ -230,7 +231,7 @@ where /// Result of [`DetachFromCollectionStatement`] when executed asynchronously pub type AsyncDetachFromCollectionResult = - Result>; + Result>; /// Errors returned by [`DetachFromCollectionStatement`]. #[derive(Debug)] diff --git a/nexus/db-queries/src/db/collection_detach_many.rs b/nexus/db-queries/src/db/collection_detach_many.rs index 0b65c404c5..8df6d4aed4 100644 --- a/nexus/db-queries/src/db/collection_detach_many.rs +++ b/nexus/db-queries/src/db/collection_detach_many.rs @@ -25,6 +25,7 @@ use diesel::prelude::*; use diesel::query_builder::*; use diesel::query_dsl::methods as query_methods; use diesel::query_source::Table; +use diesel::result::Error as DieselError; use diesel::sql_types::{Nullable, SingleValue}; use nexus_db_model::DatastoreAttachTargetConfig; use std::fmt::Debug; @@ -241,7 +242,7 @@ where /// Result of [`DetachManyFromCollectionStatement`] when executed asynchronously pub type AsyncDetachManyFromCollectionResult = - Result>; + Result>; /// Errors returned by [`DetachManyFromCollectionStatement`]. #[derive(Debug)] @@ -918,9 +919,8 @@ mod test { .set(resource::dsl::collection_id.eq(Option::::None)), ); - type TxnError = TransactionError< - DetachManyError, - >; + type TxnError = + TransactionError>; let result = conn .transaction_async(|conn| async move { detach_query.detach_and_get_result_async(&conn).await.map_err( diff --git a/nexus/db-queries/src/db/collection_insert.rs b/nexus/db-queries/src/db/collection_insert.rs index 993f16e048..b295f0574d 100644 --- a/nexus/db-queries/src/db/collection_insert.rs +++ b/nexus/db-queries/src/db/collection_insert.rs @@ -10,7 +10,7 @@ //! 3) inserts the child resource row use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionError}; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::associations::HasTable; use diesel::helper_types::*; use diesel::pg::Pg; @@ -18,6 +18,7 @@ use diesel::prelude::*; use diesel::query_builder::*; use diesel::query_dsl::methods as query_methods; use diesel::query_source::Table; +use diesel::result::Error as DieselError; use diesel::sql_types::SingleValue; use nexus_db_model::DatastoreCollectionConfig; use std::fmt::Debug; @@ -170,7 +171,7 @@ pub enum AsyncInsertError { /// The collection that the query was inserting into does not exist CollectionNotFound, /// Other database error - DatabaseError(ConnectionError), + DatabaseError(DieselError), } impl InsertIntoCollectionStatement @@ -238,14 +239,11 @@ where /// Translate from diesel errors into AsyncInsertError, handling the /// intentional division-by-zero error in the CTE. - fn translate_async_error(err: ConnectionError) -> AsyncInsertError { - match err { - ConnectionError::Query(err) - if Self::error_is_division_by_zero(&err) => - { - AsyncInsertError::CollectionNotFound - } - other => AsyncInsertError::DatabaseError(other), + fn translate_async_error(err: DieselError) -> AsyncInsertError { + if Self::error_is_division_by_zero(&err) { + AsyncInsertError::CollectionNotFound + } else { + AsyncInsertError::DatabaseError(err) } } } diff --git a/nexus/db-queries/src/db/datastore/address_lot.rs b/nexus/db-queries/src/db/datastore/address_lot.rs index 9d264dbf6b..97dfb59eba 100644 --- a/nexus/db-queries/src/db/datastore/address_lot.rs +++ b/nexus/db-queries/src/db/datastore/address_lot.rs @@ -13,9 +13,7 @@ use crate::db::error::TransactionError; use crate::db::model::Name; use crate::db::model::{AddressLot, AddressLotBlock, AddressLotReservedBlock}; use crate::db::pagination::paginated; -use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, Connection, ConnectionError, -}; +use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, Connection}; use chrono::Utc; use diesel::result::Error as DieselError; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; @@ -84,15 +82,13 @@ impl DataStore { }) .await .map_err(|e| match e { - ConnectionError::Query(DieselError::DatabaseError(_, _)) => { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::AddressLot, - ¶ms.identity.name.as_str(), - ), - ) - } + DieselError::DatabaseError(_, _) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AddressLot, + ¶ms.identity.name.as_str(), + ), + ), _ => public_error_from_diesel(e, ErrorHandler::Server), }) } @@ -151,7 +147,7 @@ impl DataStore { }) .await .map_err(|e| match e { - TxnError::Connection(e) => { + TxnError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } TxnError::CustomError(AddressLotDeleteError::LotInUse) => { @@ -252,11 +248,10 @@ pub(crate) async fn try_reserve_block( .limit(1) .first_async::(conn) .await - .map_err(|e| match e { - ConnectionError::Query(_) => ReserveBlockTxnError::CustomError( + .map_err(|_e| { + ReserveBlockTxnError::CustomError( ReserveBlockError::AddressNotInLot, - ), - e => e.into(), + ) })?; // Ensure the address is not already taken. diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs index 181b3c1798..9e4e8b1a48 100644 --- a/nexus/db-queries/src/db/datastore/db_metadata.rs +++ b/nexus/db-queries/src/db/datastore/db_metadata.rs @@ -351,7 +351,7 @@ impl DataStore { match result { Ok(()) => Ok(()), Err(TransactionError::CustomError(())) => panic!("No custom error"), - Err(TransactionError::Connection(e)) => { + Err(TransactionError::Database(e)) => { Err(public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/device_auth.rs b/nexus/db-queries/src/db/datastore/device_auth.rs index e084834833..e1facb43f6 100644 --- a/nexus/db-queries/src/db/datastore/device_auth.rs +++ b/nexus/db-queries/src/db/datastore/device_auth.rs @@ -103,7 +103,7 @@ impl DataStore { TxnError::CustomError(TokenGrantError::TooManyRequests) => { Error::internal_error("unexpectedly found multiple device auth requests for the same user code") } - TxnError::Connection(e) => { + TxnError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } }) diff --git a/nexus/db-queries/src/db/datastore/dns.rs b/nexus/db-queries/src/db/datastore/dns.rs index d9704594b1..f7ad97593e 100644 --- a/nexus/db-queries/src/db/datastore/dns.rs +++ b/nexus/db-queries/src/db/datastore/dns.rs @@ -395,7 +395,7 @@ impl DataStore { match result { Ok(()) => Ok(()), Err(TransactionError::CustomError(e)) => Err(e), - Err(TransactionError::Connection(e)) => { + Err(TransactionError::Database(e)) => { Err(public_error_from_diesel(e, ErrorHandler::Server)) } } diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs index 268b284a0a..e663130a84 100644 --- a/nexus/db-queries/src/db/datastore/external_ip.rs +++ b/nexus/db-queries/src/db/datastore/external_ip.rs @@ -143,10 +143,9 @@ impl DataStore { ) -> CreateResult { let explicit_ip = data.explicit_ip().is_some(); NextExternalIp::new(data).get_result_async(conn).await.map_err(|e| { - use async_bb8_diesel::ConnectionError::Query; use diesel::result::Error::NotFound; match e { - Query(NotFound) => { + NotFound => { if explicit_ip { Error::invalid_request( "Requested external IP address not available", diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs index bd3148f2f7..fb300ef833 100644 --- a/nexus/db-queries/src/db/datastore/ip_pool.rs +++ b/nexus/db-queries/src/db/datastore/ip_pool.rs @@ -10,7 +10,6 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::diesel_result_optional; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::fixed_data::silo::INTERNAL_SILO_ID; @@ -183,18 +182,17 @@ impl DataStore { opctx.authorize(authz::Action::Delete, authz_pool).await?; // Verify there are no IP ranges still in this pool - let range = diesel_result_optional( - ip_pool_range::dsl::ip_pool_range - .filter(ip_pool_range::dsl::ip_pool_id.eq(authz_pool.id())) - .filter(ip_pool_range::dsl::time_deleted.is_null()) - .select(ip_pool_range::dsl::id) - .limit(1) - .first_async::( - &*self.pool_connection_authorized(opctx).await?, - ) - .await, - ) - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + let range = ip_pool_range::dsl::ip_pool_range + .filter(ip_pool_range::dsl::ip_pool_id.eq(authz_pool.id())) + .filter(ip_pool_range::dsl::time_deleted.is_null()) + .select(ip_pool_range::dsl::id) + .limit(1) + .first_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if range.is_some() { return Err(Error::InvalidRequest { message: @@ -313,7 +311,6 @@ impl DataStore { .insert_and_get_result_async(conn) .await .map_err(|e| { - use async_bb8_diesel::ConnectionError::Query; use diesel::result::Error::NotFound; match e { @@ -323,7 +320,7 @@ impl DataStore { lookup_type: LookupType::ById(pool_id), } } - AsyncInsertError::DatabaseError(Query(NotFound)) => { + AsyncInsertError::DatabaseError(NotFound) => { // We've filtered out the IP addresses the client provided, // i.e., there's some overlap with existing addresses. Error::invalid_request( @@ -363,26 +360,25 @@ impl DataStore { // concurrent inserts of new external IPs from the target range by // comparing the rcgen. let conn = self.pool_connection_authorized(opctx).await?; - let range = diesel_result_optional( - dsl::ip_pool_range - .filter(dsl::ip_pool_id.eq(pool_id)) - .filter(dsl::first_address.eq(first_net)) - .filter(dsl::last_address.eq(last_net)) - .filter(dsl::time_deleted.is_null()) - .select(IpPoolRange::as_select()) - .get_result_async::(&*conn) - .await, - ) - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? - .ok_or_else(|| { - Error::invalid_request( - format!( - "The provided range {}-{} does not exist", - first_address, last_address, + let range = dsl::ip_pool_range + .filter(dsl::ip_pool_id.eq(pool_id)) + .filter(dsl::first_address.eq(first_net)) + .filter(dsl::last_address.eq(last_net)) + .filter(dsl::time_deleted.is_null()) + .select(IpPoolRange::as_select()) + .get_result_async::(&*conn) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .ok_or_else(|| { + Error::invalid_request( + format!( + "The provided range {}-{} does not exist", + first_address, last_address, + ) + .as_str(), ) - .as_str(), - ) - })?; + })?; // Find external IPs allocated out of this pool and range. let range_id = range.id; diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index b1f3203c60..7d5e32cad9 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -1670,7 +1670,6 @@ mod test { async fn test_external_ip_check_constraints() { use crate::db::model::IpKind; use crate::db::schema::external_ip::dsl; - use async_bb8_diesel::ConnectionError::Query; use diesel::result::DatabaseErrorKind::CheckViolation; use diesel::result::Error::DatabaseError; @@ -1756,10 +1755,10 @@ mod test { assert!( matches!( err, - Query(DatabaseError( + DatabaseError( CheckViolation, _ - )) + ) ), "Expected a CHECK violation when inserting a \ Floating IP record with NULL name and/or description", @@ -1805,10 +1804,10 @@ mod test { assert!( matches!( err, - Query(DatabaseError( + DatabaseError( CheckViolation, _ - )) + ) ), "Expected a CHECK violation when inserting an \ Ephemeral Service IP", @@ -1836,10 +1835,10 @@ mod test { assert!( matches!( err, - Query(DatabaseError( + DatabaseError( CheckViolation, _ - )) + ) ), "Expected a CHECK violation when inserting a \ {:?} IP record with non-NULL name, description, \ diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 3d7b8afa71..4a46b23529 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -29,6 +29,7 @@ use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; +use diesel::result::Error as DieselError; use omicron_common::api::external; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::DeleteResult; @@ -463,7 +464,7 @@ impl DataStore { #[derive(Debug)] enum NetworkInterfaceUpdateError { InstanceNotStopped, - FailedToUnsetPrimary(async_bb8_diesel::ConnectionError), + FailedToUnsetPrimary(DieselError), } type TxnError = TransactionError; diff --git a/nexus/db-queries/src/db/datastore/project.rs b/nexus/db-queries/src/db/datastore/project.rs index 0285679cd5..c447b5bf98 100644 --- a/nexus/db-queries/src/db/datastore/project.rs +++ b/nexus/db-queries/src/db/datastore/project.rs @@ -11,7 +11,6 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::diesel_result_optional; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; @@ -60,16 +59,15 @@ macro_rules! generate_fn_to_ensure_none_in_project { ) -> DeleteResult { use db::schema::$i; - let maybe_label = diesel_result_optional( - $i::dsl::$i - .filter($i::dsl::project_id.eq(authz_project.id())) - .filter($i::dsl::time_deleted.is_null()) - .select($i::dsl::$label) - .limit(1) - .first_async::<$label_ty>(&*self.pool_connection_authorized(opctx).await?) - .await, - ) - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + let maybe_label = $i::dsl::$i + .filter($i::dsl::project_id.eq(authz_project.id())) + .filter($i::dsl::time_deleted.is_null()) + .select($i::dsl::$label) + .limit(1) + .first_async::<$label_ty>(&*self.pool_connection_authorized(opctx).await?) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if let Some(label) = maybe_label { let object = stringify!($i).replace('_', " "); @@ -193,7 +191,7 @@ impl DataStore { .await .map_err(|e| match e { TransactionError::CustomError(e) => e, - TransactionError::Connection(e) => { + TransactionError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } })?; @@ -270,7 +268,7 @@ impl DataStore { .await .map_err(|e| match e { TxnError::CustomError(e) => e, - TxnError::Connection(e) => { + TxnError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } })?; diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 1be3e1ee4c..f5f7524aab 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -30,6 +30,7 @@ use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; +use diesel::result::Error as DieselError; use diesel::upsert::excluded; use nexus_db_model::DnsGroup; use nexus_db_model::DnsZone; @@ -79,7 +80,7 @@ enum RackInitError { AddingNic(Error), ServiceInsert(Error), DatasetInsert { err: AsyncInsertError, zpool_id: Uuid }, - RackUpdate { err: async_bb8_diesel::ConnectionError, rack_id: Uuid }, + RackUpdate { err: DieselError, rack_id: Uuid }, DnsSerialization(Error), Silo(Error), RoleAssignment(Error), @@ -137,7 +138,7 @@ impl From for Error { err )) } - TxnError::Connection(e) => { + TxnError::Database(e) => { Error::internal_error(&format!("Transaction error: {}", e)) } } @@ -631,7 +632,7 @@ impl DataStore { .await .map_err(|error: TxnError| match error { TransactionError::CustomError(err) => err, - TransactionError::Connection(e) => { + TransactionError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } }) diff --git a/nexus/db-queries/src/db/datastore/region_snapshot.rs b/nexus/db-queries/src/db/datastore/region_snapshot.rs index 148cfe4812..3d328a6206 100644 --- a/nexus/db-queries/src/db/datastore/region_snapshot.rs +++ b/nexus/db-queries/src/db/datastore/region_snapshot.rs @@ -10,8 +10,8 @@ use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::RegionSnapshot; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::OptionalExtension; use diesel::prelude::*; +use diesel::OptionalExtension; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::LookupResult; diff --git a/nexus/db-queries/src/db/datastore/role.rs b/nexus/db-queries/src/db/datastore/role.rs index f1198c239b..b2ad441475 100644 --- a/nexus/db-queries/src/db/datastore/role.rs +++ b/nexus/db-queries/src/db/datastore/role.rs @@ -280,7 +280,7 @@ impl DataStore { .await .map_err(|e| match e { TransactionError::CustomError(e) => e, - TransactionError::Connection(e) => { + TransactionError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } }) diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs index 5e909b84c4..ec3658c067 100644 --- a/nexus/db-queries/src/db/datastore/silo.rs +++ b/nexus/db-queries/src/db/datastore/silo.rs @@ -10,7 +10,6 @@ use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::datastore::RunnableQuery; -use crate::db::error::diesel_result_optional; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; @@ -261,7 +260,7 @@ impl DataStore { .await .map_err(|e| match e { TransactionError::CustomError(e) => e, - TransactionError::Connection(e) => { + TransactionError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } }) @@ -338,16 +337,15 @@ impl DataStore { // Make sure there are no projects present within this silo. let id = authz_silo.id(); let rcgen = db_silo.rcgen; - let project_found = diesel_result_optional( - project::dsl::project - .filter(project::dsl::silo_id.eq(id)) - .filter(project::dsl::time_deleted.is_null()) - .select(project::dsl::id) - .limit(1) - .first_async::(&*conn) - .await, - ) - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + let project_found = project::dsl::project + .filter(project::dsl::silo_id.eq(id)) + .filter(project::dsl::time_deleted.is_null()) + .select(project::dsl::id) + .limit(1) + .first_async::(&*conn) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; if project_found.is_some() { return Err(Error::InvalidRequest { @@ -395,7 +393,7 @@ impl DataStore { .await .map_err(|e| match e { TxnError::CustomError(e) => e, - TxnError::Connection(e) => { + TxnError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } })?; diff --git a/nexus/db-queries/src/db/datastore/silo_group.rs b/nexus/db-queries/src/db/datastore/silo_group.rs index d13986bb2d..46f4aae7c9 100644 --- a/nexus/db-queries/src/db/datastore/silo_group.rs +++ b/nexus/db-queries/src/db/datastore/silo_group.rs @@ -15,8 +15,8 @@ use crate::db::error::TransactionError; use crate::db::model::SiloGroup; use crate::db::model::SiloGroupMembership; use crate::db::pagination::paginated; +use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::{AsyncConnection, OptionalExtension}; use chrono::Utc; use diesel::prelude::*; use omicron_common::api::external::CreateResult; @@ -237,8 +237,7 @@ impl DataStore { "group {0} still has memberships", id )), - - TxnError::Connection(error) => { + TxnError::Database(error) => { public_error_from_diesel(error, ErrorHandler::Server) } }) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index ec6cca0071..a52d1b7772 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -183,7 +183,7 @@ impl DataStore { "No sleds can fit the requested instance", ) } - TxnError::Connection(e) => { + TxnError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } }) diff --git a/nexus/db-queries/src/db/datastore/snapshot.rs b/nexus/db-queries/src/db/datastore/snapshot.rs index 29fbb38e88..59fb00c84d 100644 --- a/nexus/db-queries/src/db/datastore/snapshot.rs +++ b/nexus/db-queries/src/db/datastore/snapshot.rs @@ -22,10 +22,9 @@ use crate::db::update_and_check::UpdateAndCheck; use crate::db::TransactionError; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::ConnectionError; use chrono::Utc; use diesel::prelude::*; -use diesel::result::Error as DieselError; +use diesel::OptionalExtension; use nexus_types::identity::Resource; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; @@ -101,7 +100,7 @@ impl DataStore { // does not match, but a project and name that does, return // ObjectAlreadyExists here. - let existing_snapshot_id: Option = match dsl::snapshot + let existing_snapshot_id: Option = dsl::snapshot .filter(dsl::time_deleted.is_null()) .filter(dsl::name.eq(snapshot.name().to_string())) .filter(dsl::project_id.eq(snapshot.project_id)) @@ -109,13 +108,7 @@ impl DataStore { .limit(1) .first_async(&conn) .await - { - Ok(v) => Ok(Some(v)), - Err(ConnectionError::Query(DieselError::NotFound)) => { - Ok(None) - } - Err(e) => Err(e), - }?; + .optional()?; if let Some(existing_snapshot_id) = existing_snapshot_id { if existing_snapshot_id != snapshot.id() { @@ -161,8 +154,7 @@ impl DataStore { } }, }, - - TxnError::Connection(e) => { + TxnError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } })?; diff --git a/nexus/db-queries/src/db/datastore/switch_interface.rs b/nexus/db-queries/src/db/datastore/switch_interface.rs index 498064ce37..88cff50471 100644 --- a/nexus/db-queries/src/db/datastore/switch_interface.rs +++ b/nexus/db-queries/src/db/datastore/switch_interface.rs @@ -14,7 +14,7 @@ use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::model::LoopbackAddress; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, ConnectionError}; +use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; use diesel::result::Error as DieselError; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use ipnetwork::IpNetwork; @@ -65,8 +65,8 @@ impl DataStore { LoopbackAddressCreateError::ReserveBlock(err), ) } - ReserveBlockTxnError::Connection(err) => { - TxnError::Connection(err) + ReserveBlockTxnError::Database(err) => { + TxnError::Database(err) } })?; @@ -103,16 +103,14 @@ impl DataStore { ReserveBlockError::AddressNotInLot, ), ) => Error::invalid_request("address not in lot"), - TxnError::Connection(e) => match e { - ConnectionError::Query(DieselError::DatabaseError(_, _)) => { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::LoopbackAddress, - &format!("lo {}", inet), - ), - ) - } + TxnError::Database(e) => match e { + DieselError::DatabaseError(_, _) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::LoopbackAddress, + &format!("lo {}", inet), + ), + ), _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index 940fedb473..45be594be6 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -20,7 +20,7 @@ use crate::db::model::{ SwitchVlanInterfaceConfig, }; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, ConnectionError}; +use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; use diesel::result::Error as DieselError; use diesel::{ ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl, @@ -279,11 +279,10 @@ impl DataStore { .limit(1) .first_async::(&conn) .await - .map_err(|e| match e { - ConnectionError::Query(_) => TxnError::CustomError( + .map_err(|_| { + TxnError::CustomError( SwitchPortSettingsCreateError::BgpAnnounceSetNotFound, - ), - e => e.into(), + ) })? } }; @@ -300,12 +299,11 @@ impl DataStore { .limit(1) .first_async::(&conn) .await - .map_err(|e| match e { - ConnectionError::Query(_) => TxnError::CustomError( + .map_err(|_| + TxnError::CustomError( SwitchPortSettingsCreateError::BgpConfigNotFound, - ), - e => e.into(), - })? + ) + )? } }; @@ -341,14 +339,11 @@ impl DataStore { .limit(1) .first_async::(&conn) .await - .map_err(|e| match e { - ConnectionError::Query(_) => { - TxnError::CustomError( - SwitchPortSettingsCreateError::AddressLotNotFound, - ) - } - e => e.into() - })? + .map_err(|_| + TxnError::CustomError( + SwitchPortSettingsCreateError::AddressLotNotFound, + ) + )? } }; // TODO: Reduce DB round trips needed for reserving ip blocks @@ -369,7 +364,7 @@ impl DataStore { SwitchPortSettingsCreateError::ReserveBlock(err) ) } - ReserveBlockTxnError::Connection(err) => TxnError::Connection(err), + ReserveBlockTxnError::Database(err) => TxnError::Database(err), })?; address_config.push(SwitchPortAddressConfig::new( @@ -416,10 +411,8 @@ impl DataStore { ReserveBlockError::AddressNotInLot ) ) => Error::invalid_request("address not in lot"), - TxnError::Connection(e) => match e { - ConnectionError::Query( - DieselError::DatabaseError(_, _), - ) => public_error_from_diesel( + TxnError::Database(e) => match e { + DieselError::DatabaseError(_, _) => public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SwitchPortSettings, @@ -467,12 +460,11 @@ impl DataStore { .limit(1) .first_async::(&conn) .await - .map_err(|e| match e { - ConnectionError::Query(_) => TxnError::CustomError( + .map_err(|_| + TxnError::CustomError( SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound, - ), - e => e.into() - })? + ) + )? } }; @@ -599,10 +591,8 @@ impl DataStore { SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound) => { Error::invalid_request("port settings not found") } - TxnError::Connection(e) => match e { - ConnectionError::Query( - DieselError::DatabaseError(_, _), - ) => { + TxnError::Database(e) => match e { + DieselError::DatabaseError(_, _) => { let name = match ¶ms.port_settings { Some(name_or_id) => name_or_id.to_string(), None => String::new(), @@ -676,11 +666,10 @@ impl DataStore { .limit(1) .first_async::(&conn) .await - .map_err(|e| match e { - ConnectionError::Query(_) => TxnError::CustomError( + .map_err(|_| { + TxnError::CustomError( SwitchPortSettingsGetError::NotFound(name.clone()) - ), - e => e.into() + ) })? } }; @@ -804,10 +793,8 @@ impl DataStore { SwitchPortSettingsGetError::NotFound(name)) => { Error::not_found_by_name(ResourceType::SwitchPortSettings, &name) } - TxnError::Connection(e) => match e { - ConnectionError::Query( - DieselError::DatabaseError(_, _), - ) => { + TxnError::Database(e) => match e { + DieselError::DatabaseError(_, _) => { let name = name_or_id.to_string(); public_error_from_diesel( e, @@ -855,11 +842,8 @@ impl DataStore { .limit(1) .first_async::(&conn) .await - .map_err(|e| match e { - ConnectionError::Query(_) => TxnError::CustomError( - SwitchPortCreateError::RackNotFound, - ), - e => e.into(), + .map_err(|_| { + TxnError::CustomError(SwitchPortCreateError::RackNotFound) })?; // insert switch port @@ -878,19 +862,14 @@ impl DataStore { TxnError::CustomError(SwitchPortCreateError::RackNotFound) => { Error::invalid_request("rack not found") } - TxnError::Connection(e) => match e { - ConnectionError::Query(DieselError::DatabaseError(_, _)) => { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::SwitchPort, - &format!( - "{}/{}/{}", - rack_id, &switch_location, &port, - ), - ), - ) - } + TxnError::Database(e) => match e { + DieselError::DatabaseError(_, _) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::SwitchPort, + &format!("{}/{}/{}", rack_id, &switch_location, &port,), + ), + ), _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) @@ -929,11 +908,8 @@ impl DataStore { .limit(1) .first_async::(&conn) .await - .map_err(|e| match e { - ConnectionError::Query(_) => { - TxnError::CustomError(SwitchPortDeleteError::NotFound) - } - e => e.into(), + .map_err(|_| { + TxnError::CustomError(SwitchPortDeleteError::NotFound) })?; if port.port_settings_id.is_some() { @@ -958,7 +934,7 @@ impl DataStore { TxnError::CustomError(SwitchPortDeleteError::ActiveSettings) => { Error::invalid_request("must clear port settings first") } - TxnError::Connection(e) => { + TxnError::Database(e) => { public_error_from_diesel(e, ErrorHandler::Server) } }) diff --git a/nexus/db-queries/src/db/datastore/update.rs b/nexus/db-queries/src/db/datastore/update.rs index 5a3e3b27e4..8b1eecb781 100644 --- a/nexus/db-queries/src/db/datastore/update.rs +++ b/nexus/db-queries/src/db/datastore/update.rs @@ -164,7 +164,7 @@ impl DataStore { .await .map_err(|e| match e { TransactionError::CustomError(e) => e, - TransactionError::Connection(e) => public_error_from_diesel( + TransactionError::Database(e) => public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::ComponentUpdate, diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index b97b8451cf..38e3875036 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -16,9 +16,9 @@ use crate::db::model::RegionSnapshot; use crate::db::model::Volume; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use async_bb8_diesel::OptionalExtension; use chrono::Utc; use diesel::prelude::*; +use diesel::OptionalExtension; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; @@ -336,7 +336,7 @@ impl DataStore { .await .map_err(|e| match e { TxnError::CustomError(VolumeGetError::DieselError(e)) => { - public_error_from_diesel(e.into(), ErrorHandler::Server) + public_error_from_diesel(e, ErrorHandler::Server) } _ => { @@ -757,7 +757,7 @@ impl DataStore { .map_err(|e| match e { TxnError::CustomError( DecreaseCrucibleResourcesError::DieselError(e), - ) => public_error_from_diesel(e.into(), ErrorHandler::Server), + ) => public_error_from_diesel(e, ErrorHandler::Server), _ => { Error::internal_error(&format!("Transaction error: {}", e)) @@ -955,7 +955,7 @@ impl DataStore { TxnError::CustomError( RemoveReadOnlyParentError::DieselError(e), ) => public_error_from_diesel( - e.into(), + e, ErrorHandler::Server, ), diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index af7ea93456..46c3d2504e 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -10,7 +10,6 @@ use crate::context::OpContext; use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; -use crate::db::error::diesel_result_optional; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; @@ -389,19 +388,18 @@ impl DataStore { // but we can't have NICs be a child of both tables at this point, and // we need to prevent VPC Subnets from being deleted while they have // NICs in them as well. - if diesel_result_optional( - vpc_subnet::dsl::vpc_subnet - .filter(vpc_subnet::dsl::vpc_id.eq(authz_vpc.id())) - .filter(vpc_subnet::dsl::time_deleted.is_null()) - .select(vpc_subnet::dsl::id) - .limit(1) - .first_async::( - &*self.pool_connection_authorized(opctx).await?, - ) - .await, - ) - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? - .is_some() + if vpc_subnet::dsl::vpc_subnet + .filter(vpc_subnet::dsl::vpc_id.eq(authz_vpc.id())) + .filter(vpc_subnet::dsl::time_deleted.is_null()) + .select(vpc_subnet::dsl::id) + .limit(1) + .first_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .is_some() { return Err(Error::InvalidRequest { message: String::from( @@ -556,7 +554,7 @@ impl DataStore { TxnError::CustomError( FirewallUpdateError::CollectionNotFound, ) => Error::not_found_by_id(ResourceType::Vpc, &authz_vpc.id()), - TxnError::Connection(e) => public_error_from_diesel( + TxnError::Database(e) => public_error_from_diesel( e, ErrorHandler::NotFoundByResource(authz_vpc), ), @@ -700,17 +698,16 @@ impl DataStore { let conn = self.pool_connection_authorized(opctx).await?; // Verify there are no child network interfaces in this VPC Subnet - if diesel_result_optional( - network_interface::dsl::network_interface - .filter(network_interface::dsl::subnet_id.eq(authz_subnet.id())) - .filter(network_interface::dsl::time_deleted.is_null()) - .select(network_interface::dsl::id) - .limit(1) - .first_async::(&*conn) - .await, - ) - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? - .is_some() + if network_interface::dsl::network_interface + .filter(network_interface::dsl::subnet_id.eq(authz_subnet.id())) + .filter(network_interface::dsl::time_deleted.is_null()) + .select(network_interface::dsl::id) + .limit(1) + .first_async::(&*conn) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .is_some() { return Err(Error::InvalidRequest { message: String::from( diff --git a/nexus/db-queries/src/db/error.rs b/nexus/db-queries/src/db/error.rs index f7402bb8c7..cbe2b0a71f 100644 --- a/nexus/db-queries/src/db/error.rs +++ b/nexus/db-queries/src/db/error.rs @@ -4,7 +4,6 @@ //! Error handling and conversions. -use async_bb8_diesel::ConnectionError; use diesel::result::DatabaseErrorInformation; use diesel::result::DatabaseErrorKind as DieselErrorKind; use diesel::result::Error as DieselError; @@ -25,16 +24,8 @@ pub enum TransactionError { /// /// This error covers failure due to accessing the DB pool or errors /// propagated from the DB itself. - #[error("Connection error: {0}")] - Connection(#[from] async_bb8_diesel::ConnectionError), -} - -// Maps a "diesel error" into a "pool error", which -// is already contained within the error type. -impl From for TransactionError { - fn from(err: DieselError) -> Self { - Self::Connection(ConnectionError::Query(err)) - } + #[error("Database error: {0}")] + Database(#[from] DieselError), } impl From for TransactionError { @@ -50,8 +41,9 @@ impl TransactionError { /// [1]: https://www.cockroachlabs.com/docs/v23.1/transaction-retry-error-reference#client-side-retry-handling pub fn retry_transaction(&self) -> bool { match &self { - TransactionError::Connection(ConnectionError::Query( - DieselError::DatabaseError(kind, boxed_error_information), + Self::Database(DieselError::DatabaseError( + kind, + boxed_error_information, )) => match kind { DieselErrorKind::SerializationFailure => { return boxed_error_information @@ -93,19 +85,6 @@ fn format_database_error( rv } -/// Like [`diesel::result::OptionalExtension::optional`]. This turns Ok(v) -/// into Ok(Some(v)), Err("NotFound") into Ok(None), and leave all other values -/// unchanged. -pub fn diesel_result_optional( - result: Result, -) -> Result, ConnectionError> { - match result { - Ok(v) => Ok(Some(v)), - Err(ConnectionError::Query(DieselError::NotFound)) => Ok(None), - Err(e) => Err(e), - } -} - /// Allows the caller to handle user-facing errors, and provide additional /// context which may be used to populate more informative errors. /// @@ -142,41 +121,27 @@ pub enum ErrorHandler<'a> { /// [`ErrorHandler`] may be used to add additional handlers for the error /// being returned. pub fn public_error_from_diesel( - error: ConnectionError, + error: DieselError, handler: ErrorHandler<'_>, ) -> PublicError { - match error { - ConnectionError::Connection(error) => PublicError::unavail(&format!( - "Failed to access connection pool: {}", + match handler { + ErrorHandler::NotFoundByResource(resource) => { + public_error_from_diesel_lookup( + error, + resource.resource_type(), + resource.lookup_type(), + ) + } + ErrorHandler::NotFoundByLookup(resource_type, lookup_type) => { + public_error_from_diesel_lookup(error, resource_type, &lookup_type) + } + ErrorHandler::Conflict(resource_type, object_name) => { + public_error_from_diesel_create(error, resource_type, object_name) + } + ErrorHandler::Server => PublicError::internal_error(&format!( + "unexpected database error: {:#}", error )), - ConnectionError::Query(error) => match handler { - ErrorHandler::NotFoundByResource(resource) => { - public_error_from_diesel_lookup( - error, - resource.resource_type(), - resource.lookup_type(), - ) - } - ErrorHandler::NotFoundByLookup(resource_type, lookup_type) => { - public_error_from_diesel_lookup( - error, - resource_type, - &lookup_type, - ) - } - ErrorHandler::Conflict(resource_type, object_name) => { - public_error_from_diesel_create( - error, - resource_type, - object_name, - ) - } - ErrorHandler::Server => PublicError::internal_error(&format!( - "unexpected database error: {:#}", - error - )), - }, } } diff --git a/nexus/db-queries/src/db/explain.rs b/nexus/db-queries/src/db/explain.rs index fc8098b876..3de5b4f280 100644 --- a/nexus/db-queries/src/db/explain.rs +++ b/nexus/db-queries/src/db/explain.rs @@ -5,11 +5,12 @@ //! Utility allowing Diesel to EXPLAIN queries. use super::pool::DbConnection; -use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionError}; +use async_bb8_diesel::AsyncRunQueryDsl; use async_trait::async_trait; use diesel::pg::Pg; use diesel::prelude::*; use diesel::query_builder::*; +use diesel::result::Error as DieselError; /// A wrapper around a runnable Diesel query, which EXPLAINs what it is doing. /// @@ -49,7 +50,7 @@ pub trait ExplainableAsync { async fn explain_async( self, conn: &async_bb8_diesel::Connection, - ) -> Result; + ) -> Result; } #[async_trait] @@ -65,7 +66,7 @@ where async fn explain_async( self, conn: &async_bb8_diesel::Connection, - ) -> Result { + ) -> Result { Ok(ExplainStatement { query: self } .get_results_async::(conn) .await? diff --git a/nexus/db-queries/src/db/pool.rs b/nexus/db-queries/src/db/pool.rs index 6311121bd1..73c95f4e91 100644 --- a/nexus/db-queries/src/db/pool.rs +++ b/nexus/db-queries/src/db/pool.rs @@ -99,7 +99,9 @@ impl CustomizeConnection, ConnectionError> &self, conn: &mut Connection, ) -> Result<(), ConnectionError> { - conn.batch_execute_async(DISALLOW_FULL_TABLE_SCAN_SQL).await + conn.batch_execute_async(DISALLOW_FULL_TABLE_SCAN_SQL) + .await + .map_err(|e| e.into()) } } diff --git a/nexus/db-queries/src/db/queries/external_ip.rs b/nexus/db-queries/src/db/queries/external_ip.rs index 18360e1045..cf182e080d 100644 --- a/nexus/db-queries/src/db/queries/external_ip.rs +++ b/nexus/db-queries/src/db/queries/external_ip.rs @@ -20,6 +20,7 @@ use diesel::query_builder::AstPass; use diesel::query_builder::Query; use diesel::query_builder::QueryFragment; use diesel::query_builder::QueryId; +use diesel::result::Error as DieselError; use diesel::sql_types; use diesel::Column; use diesel::Expression; @@ -42,7 +43,7 @@ const REALLOCATION_WITH_DIFFERENT_IP_SENTINEL: &'static str = "Reallocation of IP with different value"; /// Translates a generic pool error to an external error. -pub fn from_diesel(e: async_bb8_diesel::ConnectionError) -> external::Error { +pub fn from_diesel(e: DieselError) -> external::Error { use crate::db::error; let sentinels = [REALLOCATION_WITH_DIFFERENT_IP_SENTINEL]; diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs index 877daad9e3..bac2610b41 100644 --- a/nexus/db-queries/src/db/queries/network_interface.rs +++ b/nexus/db-queries/src/db/queries/network_interface.rs @@ -17,6 +17,7 @@ use diesel::prelude::Column; use diesel::query_builder::AstPass; use diesel::query_builder::QueryFragment; use diesel::query_builder::QueryId; +use diesel::result::Error as DieselError; use diesel::sql_types; use diesel::Insertable; use diesel::QueryResult; @@ -126,16 +127,14 @@ impl InsertError { /// address exhaustion or an attempt to attach an interface to an instance /// that is already associated with another VPC. pub fn from_diesel( - e: async_bb8_diesel::ConnectionError, + e: DieselError, interface: &IncompleteNetworkInterface, ) -> Self { use crate::db::error; - use async_bb8_diesel::ConnectionError; - use diesel::result::Error; match e { // Catch the specific errors designed to communicate the failures we // want to distinguish - ConnectionError::Query(Error::DatabaseError(_, _)) => { + DieselError::DatabaseError(_, _) => { decode_database_error(e, interface) } // Any other error at all is a bug @@ -223,13 +222,11 @@ impl InsertError { /// As such, it naturally is extremely tightly coupled to the database itself, /// including the software version and our schema. fn decode_database_error( - err: async_bb8_diesel::ConnectionError, + err: DieselError, interface: &IncompleteNetworkInterface, ) -> InsertError { use crate::db::error; - use async_bb8_diesel::ConnectionError; use diesel::result::DatabaseErrorKind; - use diesel::result::Error; // Error message generated when we attempt to insert an interface in a // different VPC from the interface(s) already associated with the instance @@ -292,10 +289,10 @@ fn decode_database_error( // If the address allocation subquery fails, we'll attempt to insert // NULL for the `ip` column. This checks that the non-NULL constraint on // that colum has been violated. - ConnectionError::Query(Error::DatabaseError( + DieselError::DatabaseError( DatabaseErrorKind::NotNullViolation, - ref info, - )) if info.message() == IP_EXHAUSTION_ERROR_MESSAGE => { + info, + ) if info.message() == IP_EXHAUSTION_ERROR_MESSAGE => { InsertError::NoAvailableIpAddresses } @@ -303,29 +300,27 @@ fn decode_database_error( // `push_ensure_unique_vpc_expression` subquery, which generates a // UUID parsing error if the resource (e.g. instance) we want to attach // to is already associated with another VPC. - ConnectionError::Query(Error::DatabaseError( - DatabaseErrorKind::Unknown, - ref info, - )) if info.message() == MULTIPLE_VPC_ERROR_MESSAGE => { + DieselError::DatabaseError(DatabaseErrorKind::Unknown, info) + if info.message() == MULTIPLE_VPC_ERROR_MESSAGE => + { InsertError::ResourceSpansMultipleVpcs(interface.parent_id) } // This checks the constraint on the interface slot numbers, used to // limit total number of interfaces per resource to a maximum number. - ConnectionError::Query(Error::DatabaseError( - DatabaseErrorKind::CheckViolation, - ref info, - )) if info.message() == NO_SLOTS_AVAILABLE_ERROR_MESSAGE => { + DieselError::DatabaseError(DatabaseErrorKind::CheckViolation, info) + if info.message() == NO_SLOTS_AVAILABLE_ERROR_MESSAGE => + { InsertError::NoSlotsAvailable } // If the MAC allocation subquery fails, we'll attempt to insert NULL // for the `mac` column. This checks that the non-NULL constraint on // that column has been violated. - ConnectionError::Query(Error::DatabaseError( + DieselError::DatabaseError( DatabaseErrorKind::NotNullViolation, - ref info, - )) if info.message() == MAC_EXHAUSTION_ERROR_MESSAGE => { + info, + ) if info.message() == MAC_EXHAUSTION_ERROR_MESSAGE => { InsertError::NoMacAddrressesAvailable } @@ -333,39 +328,36 @@ fn decode_database_error( // `push_ensure_unique_vpc_subnet_expression` subquery, which generates // a UUID parsing error if the resource has another interface in the VPC // Subnet of the one we're trying to insert. - ConnectionError::Query(Error::DatabaseError( - DatabaseErrorKind::Unknown, - ref info, - )) if info.message() == NON_UNIQUE_VPC_SUBNET_ERROR_MESSAGE => { + DieselError::DatabaseError(DatabaseErrorKind::Unknown, info) + if info.message() == NON_UNIQUE_VPC_SUBNET_ERROR_MESSAGE => + { InsertError::NonUniqueVpcSubnets } // This catches the UUID-cast failure intentionally introduced by // `push_instance_state_verification_subquery`, which verifies that // the instance is actually stopped when running this query. - ConnectionError::Query(Error::DatabaseError( - DatabaseErrorKind::Unknown, - ref info, - )) if info.message() == INSTANCE_BAD_STATE_ERROR_MESSAGE => { + DieselError::DatabaseError(DatabaseErrorKind::Unknown, info) + if info.message() == INSTANCE_BAD_STATE_ERROR_MESSAGE => + { assert_eq!(interface.kind, NetworkInterfaceKind::Instance); InsertError::InstanceMustBeStopped(interface.parent_id) } // This catches the UUID-cast failure intentionally introduced by // `push_instance_state_verification_subquery`, which verifies that // the instance doesn't even exist when running this query. - ConnectionError::Query(Error::DatabaseError( - DatabaseErrorKind::Unknown, - ref info, - )) if info.message() == NO_INSTANCE_ERROR_MESSAGE => { + DieselError::DatabaseError(DatabaseErrorKind::Unknown, info) + if info.message() == NO_INSTANCE_ERROR_MESSAGE => + { assert_eq!(interface.kind, NetworkInterfaceKind::Instance); InsertError::InstanceNotFound(interface.parent_id) } // This path looks specifically at constraint names. - ConnectionError::Query(Error::DatabaseError( + DieselError::DatabaseError( DatabaseErrorKind::UniqueViolation, ref info, - )) => match info.constraint_name() { + ) => match info.constraint_name() { // Constraint violated if a user-requested IP address has // already been assigned within the same VPC Subnet. Some(constraint) if constraint == IP_NOT_AVAILABLE_CONSTRAINT => { @@ -1550,17 +1542,12 @@ impl DeleteError { /// can generate, specifically the intentional errors that indicate that /// either the instance is still running, or that the instance has one or /// more secondary interfaces. - pub fn from_diesel( - e: async_bb8_diesel::ConnectionError, - query: &DeleteQuery, - ) -> Self { + pub fn from_diesel(e: DieselError, query: &DeleteQuery) -> Self { use crate::db::error; - use async_bb8_diesel::ConnectionError; - use diesel::result::Error; match e { // Catch the specific errors designed to communicate the failures we // want to distinguish - ConnectionError::Query(Error::DatabaseError(_, _)) => { + DieselError::DatabaseError(_, _) => { decode_delete_network_interface_database_error( e, query.parent_id, @@ -1608,13 +1595,11 @@ impl DeleteError { /// As such, it naturally is extremely tightly coupled to the database itself, /// including the software version and our schema. fn decode_delete_network_interface_database_error( - err: async_bb8_diesel::ConnectionError, + err: DieselError, parent_id: Uuid, ) -> DeleteError { use crate::db::error; - use async_bb8_diesel::ConnectionError; use diesel::result::DatabaseErrorKind; - use diesel::result::Error; // Error message generated when we're attempting to delete a primary // interface, and that instance also has one or more secondary interfaces @@ -1627,29 +1612,26 @@ fn decode_delete_network_interface_database_error( // first CTE, which generates a UUID parsing error if we're trying to // delete the primary interface, and the instance also has one or more // secondaries. - ConnectionError::Query(Error::DatabaseError( - DatabaseErrorKind::Unknown, - ref info, - )) if info.message() == HAS_SECONDARIES_ERROR_MESSAGE => { + DieselError::DatabaseError(DatabaseErrorKind::Unknown, ref info) + if info.message() == HAS_SECONDARIES_ERROR_MESSAGE => + { DeleteError::SecondariesExist(parent_id) } // This catches the UUID-cast failure intentionally introduced by // `push_instance_state_verification_subquery`, which verifies that // the instance can be worked on when running this query. - ConnectionError::Query(Error::DatabaseError( - DatabaseErrorKind::Unknown, - ref info, - )) if info.message() == INSTANCE_BAD_STATE_ERROR_MESSAGE => { + DieselError::DatabaseError(DatabaseErrorKind::Unknown, ref info) + if info.message() == INSTANCE_BAD_STATE_ERROR_MESSAGE => + { DeleteError::InstanceBadState(parent_id) } // This catches the UUID-cast failure intentionally introduced by // `push_instance_state_verification_subquery`, which verifies that // the instance doesn't even exist when running this query. - ConnectionError::Query(Error::DatabaseError( - DatabaseErrorKind::Unknown, - ref info, - )) if info.message() == NO_INSTANCE_ERROR_MESSAGE => { + DieselError::DatabaseError(DatabaseErrorKind::Unknown, ref info) + if info.message() == NO_INSTANCE_ERROR_MESSAGE => + { DeleteError::InstanceNotFound(parent_id) } diff --git a/nexus/db-queries/src/db/queries/region_allocation.rs b/nexus/db-queries/src/db/queries/region_allocation.rs index 7f7b2ea9bf..a080af4c37 100644 --- a/nexus/db-queries/src/db/queries/region_allocation.rs +++ b/nexus/db-queries/src/db/queries/region_allocation.rs @@ -14,6 +14,7 @@ use crate::db::true_or_cast_error::{matches_sentinel, TrueOrCastError}; use db_macros::Subquery; use diesel::pg::Pg; use diesel::query_builder::{AstPass, Query, QueryFragment, QueryId}; +use diesel::result::Error as DieselError; use diesel::PgBinaryExpressionMethods; use diesel::{ sql_types, BoolExpressionMethods, Column, CombineDsl, ExpressionMethods, @@ -36,7 +37,7 @@ const NOT_ENOUGH_UNIQUE_ZPOOLS_SENTINEL: &'static str = /// Translates a generic pool error to an external error based /// on messages which may be emitted during region provisioning. -pub fn from_diesel(e: async_bb8_diesel::ConnectionError) -> external::Error { +pub fn from_diesel(e: DieselError) -> external::Error { use crate::db::error; let sentinels = [ diff --git a/nexus/db-queries/src/db/queries/vpc_subnet.rs b/nexus/db-queries/src/db/queries/vpc_subnet.rs index bbb229da1e..9ddec32080 100644 --- a/nexus/db-queries/src/db/queries/vpc_subnet.rs +++ b/nexus/db-queries/src/db/queries/vpc_subnet.rs @@ -11,6 +11,7 @@ use chrono::{DateTime, Utc}; use diesel::pg::Pg; use diesel::prelude::*; use diesel::query_builder::*; +use diesel::result::Error as DieselError; use diesel::sql_types; use omicron_common::api::external; use ref_cast::RefCast; @@ -28,14 +29,9 @@ pub enum SubnetError { impl SubnetError { /// Construct a `SubnetError` from a Diesel error, catching the desired /// cases and building useful errors. - pub fn from_diesel( - e: async_bb8_diesel::ConnectionError, - subnet: &VpcSubnet, - ) -> Self { + pub fn from_diesel(e: DieselError, subnet: &VpcSubnet) -> Self { use crate::db::error; - use async_bb8_diesel::ConnectionError; use diesel::result::DatabaseErrorKind; - use diesel::result::Error; const IPV4_OVERLAP_ERROR_MESSAGE: &str = r#"null value in column "ipv4_block" violates not-null constraint"#; const IPV6_OVERLAP_ERROR_MESSAGE: &str = @@ -43,26 +39,26 @@ impl SubnetError { const NAME_CONFLICT_CONSTRAINT: &str = "vpc_subnet_vpc_id_name_key"; match e { // Attempt to insert overlapping IPv4 subnet - ConnectionError::Query(Error::DatabaseError( + DieselError::DatabaseError( DatabaseErrorKind::NotNullViolation, ref info, - )) if info.message() == IPV4_OVERLAP_ERROR_MESSAGE => { + ) if info.message() == IPV4_OVERLAP_ERROR_MESSAGE => { SubnetError::OverlappingIpRange(subnet.ipv4_block.0 .0.into()) } // Attempt to insert overlapping IPv6 subnet - ConnectionError::Query(Error::DatabaseError( + DieselError::DatabaseError( DatabaseErrorKind::NotNullViolation, ref info, - )) if info.message() == IPV6_OVERLAP_ERROR_MESSAGE => { + ) if info.message() == IPV6_OVERLAP_ERROR_MESSAGE => { SubnetError::OverlappingIpRange(subnet.ipv6_block.0 .0.into()) } // Conflicting name for the subnet within a VPC - ConnectionError::Query(Error::DatabaseError( + DieselError::DatabaseError( DatabaseErrorKind::UniqueViolation, ref info, - )) if info.constraint_name() == Some(NAME_CONFLICT_CONSTRAINT) => { + ) if info.constraint_name() == Some(NAME_CONFLICT_CONSTRAINT) => { SubnetError::External(error::public_error_from_diesel( e, error::ErrorHandler::Conflict( diff --git a/nexus/db-queries/src/db/true_or_cast_error.rs b/nexus/db-queries/src/db/true_or_cast_error.rs index e04d865182..6d7b2a1dbd 100644 --- a/nexus/db-queries/src/db/true_or_cast_error.rs +++ b/nexus/db-queries/src/db/true_or_cast_error.rs @@ -9,6 +9,7 @@ use diesel::pg::Pg; use diesel::query_builder::AstPass; use diesel::query_builder::QueryFragment; use diesel::query_builder::QueryId; +use diesel::result::Error as DieselError; use diesel::Expression; use diesel::SelectableExpression; @@ -77,10 +78,9 @@ where /// Returns one of the sentinels if it matches the expected value from /// a [`TrueOrCastError`]. pub fn matches_sentinel( - e: &async_bb8_diesel::ConnectionError, + e: &DieselError, sentinels: &[&'static str], ) -> Option<&'static str> { - use async_bb8_diesel::ConnectionError; use diesel::result::DatabaseErrorKind; use diesel::result::Error; @@ -93,10 +93,7 @@ pub fn matches_sentinel( match e { // Catch the specific errors designed to communicate the failures we // want to distinguish. - ConnectionError::Query(Error::DatabaseError( - DatabaseErrorKind::Unknown, - ref info, - )) => { + Error::DatabaseError(DatabaseErrorKind::Unknown, info) => { for sentinel in sentinels { if info.message() == bool_parse_error(sentinel) { return Some(sentinel); diff --git a/nexus/db-queries/src/db/update_and_check.rs b/nexus/db-queries/src/db/update_and_check.rs index 96cb3e4c79..d6bf14c083 100644 --- a/nexus/db-queries/src/db/update_and_check.rs +++ b/nexus/db-queries/src/db/update_and_check.rs @@ -12,6 +12,7 @@ use diesel::prelude::*; use diesel::query_builder::*; use diesel::query_dsl::methods::LoadQuery; use diesel::query_source::Table; +use diesel::result::Error as DieselError; use diesel::sql_types::Nullable; use diesel::QuerySource; use std::marker::PhantomData; @@ -156,7 +157,7 @@ where pub async fn execute_and_check( self, conn: &async_bb8_diesel::Connection, - ) -> Result, async_bb8_diesel::ConnectionError> + ) -> Result, DieselError> where // We require this bound to ensure that "Self" is runnable as query. Self: LoadQuery<'static, DbConnection, (Option, Option, Q)>, diff --git a/nexus/src/app/sagas/disk_create.rs b/nexus/src/app/sagas/disk_create.rs index 275c8738cc..fe403a7d41 100644 --- a/nexus/src/app/sagas/disk_create.rs +++ b/nexus/src/app/sagas/disk_create.rs @@ -832,9 +832,10 @@ pub(crate) mod test { }; use async_bb8_diesel::{ AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - OptionalExtension, }; - use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; + use diesel::{ + ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, + }; use dropshot::test_util::ClientTestContext; use nexus_db_queries::context::OpContext; use nexus_db_queries::{authn::saga::Serialized, db::datastore::DataStore}; diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index 6fc93ce8db..2762ecaff3 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -1372,10 +1372,10 @@ pub mod test { }; use async_bb8_diesel::{ AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - OptionalExtension, }; use diesel::{ - BoolExpressionMethods, ExpressionMethods, QueryDsl, SelectableHelper, + BoolExpressionMethods, ExpressionMethods, OptionalExtension, QueryDsl, + SelectableHelper, }; use dropshot::test_util::ClientTestContext; use nexus_db_queries::authn::saga::Serialized; diff --git a/nexus/src/app/sagas/project_create.rs b/nexus/src/app/sagas/project_create.rs index 1cbf9070ee..135e20ff06 100644 --- a/nexus/src/app/sagas/project_create.rs +++ b/nexus/src/app/sagas/project_create.rs @@ -159,9 +159,10 @@ mod test { }; use async_bb8_diesel::{ AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - OptionalExtension, }; - use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; + use diesel::{ + ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, + }; use nexus_db_queries::{ authn::saga::Serialized, authz, context::OpContext, db::datastore::DataStore, diff --git a/nexus/src/app/sagas/snapshot_create.rs b/nexus/src/app/sagas/snapshot_create.rs index 9c8a33fb17..0b3c5c99d7 100644 --- a/nexus/src/app/sagas/snapshot_create.rs +++ b/nexus/src/app/sagas/snapshot_create.rs @@ -1568,8 +1568,10 @@ mod test { use crate::app::sagas::test_helpers; use crate::app::test_interfaces::TestInterfaces; use crate::external_api::shared::IpRange; - use async_bb8_diesel::{AsyncRunQueryDsl, OptionalExtension}; - use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; + use async_bb8_diesel::AsyncRunQueryDsl; + use diesel::{ + ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, + }; use dropshot::test_util::ClientTestContext; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index 85eed6616d..4b5bedf41e 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -445,8 +445,10 @@ pub(crate) mod test { app::saga::create_saga_dag, app::sagas::vpc_create::Params, app::sagas::vpc_create::SagaVpcCreate, external_api::params, }; - use async_bb8_diesel::{AsyncRunQueryDsl, OptionalExtension}; - use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; + use async_bb8_diesel::AsyncRunQueryDsl; + use diesel::{ + ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, + }; use dropshot::test_util::ClientTestContext; use nexus_db_queries::{ authn::saga::Serialized, authz, context::OpContext, From c6955a5a0452c958059ae1de9376389c0286c14e Mon Sep 17 00:00:00 2001 From: Rain Date: Wed, 11 Oct 2023 18:51:25 -0700 Subject: [PATCH 46/89] [dependabot] remove in favor of Renovate (#4264) For folks with access to Oxide RFDs, see https://rfd.shared.oxide.computer/rfd/0434 for more. --- .github/dependabot.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 1b94f4bd27..0000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,17 +0,0 @@ -# -# Dependabot configuration file -# - -version: 2 -updates: - - package-ecosystem: "cargo" - directory: "/" - schedule: - interval: "weekly" - open-pull-requests-limit: 20 - groups: - russh: - # russh and russh-keys must be updated in lockstep - patterns: - - "russh" - - "russh-keys" From 876e8ca7c601a79beef0a787e83ec13a6cc1db7f Mon Sep 17 00:00:00 2001 From: Greg Colombo Date: Wed, 11 Oct 2023 22:25:18 -0700 Subject: [PATCH 47/89] Split instance state into Instance and VMM tables (#4194) Refactor the definition of an `Instance` throughout the control plane so that an `Instance` is separate from the `Vmm`s that incarnate it. This confers several advantages: - VMMs have their own state that sled agent can update without necessarily changing their instance's state. It's also possible to change an instance's active Propolis ID without having to know or update an instance's Propolis IP or current sled ID, since these change when an instance's active Propolis ID changes. This removes a great deal of complexity in sled agent, especially when live migrating an instance, and also simplifies the live migration saga considerably. - Resource reservations for instances have much clearer lifetimes: a reservation can be released when its VMM has moved to a terminal state. Nexus no longer has to reason about VMM lifetimes from changes to an instance's Propolis ID columns. - It's now possible for an Instance not to have an active Propolis at all! This allows an instance not to reserve sled resources when it's not running. It also allows an instance to stop and restart on a different sled. - It's also possible to get a history of an instance's VMMs for, e.g., zone bundle examination purposes ("my VMM had a problem two days ago but it went away when I stopped and restarted it; can you investigate?"). Rework callers throughout Nexus who depend on knowing an instance's current state and/or its current sled ID. In many cases (e.g. disk and NIC attach and detach), the relevant detail is whether the instance has an active Propolis; for simplicity, augment these checks with "has an active Propolis ID" instead of trying to grab both instance and VMM states. ## Known issues/remaining work - The virtual provisioning table is still updated only at instance creation/deletion time. Usage metrics that depend on this table might report strange and wonderful values if a user creates many more instances than can be started at one time. - Instances still use the generic "resource attachment" CTE to manage attaching and detaching disks. Previously these queries looked at instance states; now they look at an instance's state and whether it has an active Propolis, but not at the active Propolis's state. This will need to be revisited in the future to support disk hotplug. - `handle_instance_put_result` is still very aggressive about setting instances to the Failed state if sled agent returns errors other than invalid-request-flavored errors. I think we should reconsider this behavior, but this change is big enough as it is. I will file a TODO for this and update the new comments accordingly before this merges. - The new live migration logic is not tested yet and differs from the "two-table" TLA+ model in RFD 361. More work will be needed here before we can declare live migration fully ready for selfhosting. - It would be nice to have an `omdb vmm` command; for now I've just updated existing `omdb` commands to deal with the optionality of Propolises and sleds. Tests: - Unit/integration tests - On a single-machine dev cluster, created two instances and verified that: - The instances only have resource reservations while they're running (and they reserve reservoir space now) - The instances can reach each other over their internal and external IPs when they're both running (and can still reach each other even if you try to delete one while it's active) - `scadm` shows the appropriate IP mappings being added/deleted as the instances start/stop - The instances' serial consoles work as expected - Attaching a new disk to an instance is only possible if the instance is stopped - Disk snapshot succeeds when invoked on a running instance's attached disk - Deleting an instance detaches its disks - `omicron-stress` on a single-machine dev cluster ran for about an hour and created ~800 instances without any instances going to the Failed state (previously this would happen in the first 5-10 minutes) --- clients/nexus-client/src/lib.rs | 43 +- clients/sled-agent-client/src/lib.rs | 38 +- common/src/api/external/mod.rs | 31 +- common/src/api/internal/nexus.rs | 69 +- dev-tools/omdb/src/bin/omdb/db.rs | 132 ++- nexus/db-model/src/instance.rs | 219 ++-- nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/schema.rs | 28 +- nexus/db-model/src/vmm.rs | 137 +++ nexus/db-queries/src/db/datastore/disk.rs | 32 +- nexus/db-queries/src/db/datastore/instance.rs | 207 +++- nexus/db-queries/src/db/datastore/mod.rs | 2 + .../src/db/datastore/network_interface.rs | 22 +- nexus/db-queries/src/db/datastore/sled.rs | 15 +- nexus/db-queries/src/db/datastore/vmm.rs | 161 +++ nexus/db-queries/src/db/datastore/vpc.rs | 9 +- nexus/db-queries/src/db/queries/instance.rs | 255 +++++ nexus/db-queries/src/db/queries/mod.rs | 1 + .../src/db/queries/network_interface.rs | 160 +-- nexus/src/app/instance.rs | 956 +++++++++++------- nexus/src/app/instance_network.rs | 210 ++++ nexus/src/app/sagas/finalize_disk.rs | 2 +- nexus/src/app/sagas/instance_common.rs | 135 +++ nexus/src/app/sagas/instance_create.rs | 530 +--------- nexus/src/app/sagas/instance_delete.rs | 151 +-- nexus/src/app/sagas/instance_migrate.rs | 501 ++++----- nexus/src/app/sagas/instance_start.rs | 546 +++++----- nexus/src/app/sagas/mod.rs | 3 +- nexus/src/app/sagas/snapshot_create.rs | 208 ++-- nexus/src/app/sagas/test_helpers.rs | 43 +- nexus/src/app/snapshot.rs | 60 +- nexus/src/app/test_interfaces.rs | 54 +- nexus/src/cidata.rs | 2 +- nexus/src/external_api/http_entrypoints.rs | 12 +- nexus/src/internal_api/http_entrypoints.rs | 4 +- nexus/tests/integration_tests/disks.rs | 6 +- nexus/tests/integration_tests/instances.rs | 374 ++++--- nexus/tests/integration_tests/ip_pools.rs | 6 +- nexus/tests/integration_tests/pantry.rs | 6 +- nexus/tests/integration_tests/schema.rs | 12 +- openapi/nexus-internal.json | 138 +-- openapi/sled-agent.json | 202 ++-- schema/crdb/6.0.0/README.adoc | 14 + schema/crdb/6.0.0/up01.sql | 6 + schema/crdb/6.0.0/up02.sql | 13 + schema/crdb/6.0.0/up03.sql | 11 + schema/crdb/6.0.0/up04.sql | 23 + schema/crdb/6.0.0/up05.sql | 8 + schema/crdb/6.0.0/up06.sql | 1 + schema/crdb/6.0.0/up07.sql | 1 + schema/crdb/6.0.0/up08.sql | 1 + schema/crdb/6.0.0/up09.sql | 10 + schema/crdb/README.adoc | 64 +- schema/crdb/dbinit.sql | 120 ++- sled-agent/src/common/instance.rs | 865 ++++++++++------ sled-agent/src/http_entrypoints.rs | 16 +- sled-agent/src/instance.rs | 301 +++--- sled-agent/src/instance_manager.rs | 59 +- sled-agent/src/params.rs | 32 +- sled-agent/src/sim/collection.rs | 191 ++-- sled-agent/src/sim/http_entrypoints.rs | 15 +- sled-agent/src/sim/instance.rs | 209 ++-- sled-agent/src/sim/sled_agent.rs | 73 +- sled-agent/src/sled_agent.rs | 24 +- 64 files changed, 4694 insertions(+), 3087 deletions(-) create mode 100644 nexus/db-model/src/vmm.rs create mode 100644 nexus/db-queries/src/db/datastore/vmm.rs create mode 100644 nexus/db-queries/src/db/queries/instance.rs create mode 100644 nexus/src/app/sagas/instance_common.rs create mode 100644 schema/crdb/6.0.0/README.adoc create mode 100644 schema/crdb/6.0.0/up01.sql create mode 100644 schema/crdb/6.0.0/up02.sql create mode 100644 schema/crdb/6.0.0/up03.sql create mode 100644 schema/crdb/6.0.0/up04.sql create mode 100644 schema/crdb/6.0.0/up05.sql create mode 100644 schema/crdb/6.0.0/up06.sql create mode 100644 schema/crdb/6.0.0/up07.sql create mode 100644 schema/crdb/6.0.0/up08.sql create mode 100644 schema/crdb/6.0.0/up09.sql diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs index 412ca70497..33a68cb3ce 100644 --- a/clients/nexus-client/src/lib.rs +++ b/clients/nexus-client/src/lib.rs @@ -88,22 +88,41 @@ impl From s: omicron_common::api::internal::nexus::InstanceRuntimeState, ) -> Self { Self { - run_state: s.run_state.into(), - sled_id: s.sled_id, - propolis_id: s.propolis_id, dst_propolis_id: s.dst_propolis_id, - propolis_addr: s.propolis_addr.map(|addr| addr.to_string()), + gen: s.gen.into(), migration_id: s.migration_id, - propolis_gen: s.propolis_gen.into(), - ncpus: s.ncpus.into(), - memory: s.memory.into(), - hostname: s.hostname, + propolis_id: s.propolis_id, + time_updated: s.time_updated, + } + } +} + +impl From + for types::VmmRuntimeState +{ + fn from(s: omicron_common::api::internal::nexus::VmmRuntimeState) -> Self { + Self { gen: s.gen.into(), + state: s.state.into(), time_updated: s.time_updated, } } } +impl From + for types::SledInstanceState +{ + fn from( + s: omicron_common::api::internal::nexus::SledInstanceState, + ) -> Self { + Self { + instance_state: s.instance_state.into(), + propolis_id: s.propolis_id, + vmm_state: s.vmm_state.into(), + } + } +} + impl From for types::InstanceState { @@ -124,14 +143,6 @@ impl From } } -impl From - for types::InstanceCpuCount -{ - fn from(s: omicron_common::api::external::InstanceCpuCount) -> Self { - Self(s.0) - } -} - impl From for types::Generation { fn from(s: omicron_common::api::external::Generation) -> Self { Self(i64::from(&s) as u64) diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 68e60e8d95..3daac7dd60 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -24,16 +24,9 @@ impl From s: omicron_common::api::internal::nexus::InstanceRuntimeState, ) -> Self { Self { - run_state: s.run_state.into(), - sled_id: s.sled_id, propolis_id: s.propolis_id, dst_propolis_id: s.dst_propolis_id, - propolis_addr: s.propolis_addr.map(|addr| addr.to_string()), migration_id: s.migration_id, - propolis_gen: s.propolis_gen.into(), - ncpus: s.ncpus.into(), - memory: s.memory.into(), - hostname: s.hostname, gen: s.gen.into(), time_updated: s.time_updated, } @@ -85,22 +78,39 @@ impl From { fn from(s: types::InstanceRuntimeState) -> Self { Self { - run_state: s.run_state.into(), - sled_id: s.sled_id, propolis_id: s.propolis_id, dst_propolis_id: s.dst_propolis_id, - propolis_addr: s.propolis_addr.map(|addr| addr.parse().unwrap()), migration_id: s.migration_id, - propolis_gen: s.propolis_gen.into(), - ncpus: s.ncpus.into(), - memory: s.memory.into(), - hostname: s.hostname, gen: s.gen.into(), time_updated: s.time_updated, } } } +impl From + for omicron_common::api::internal::nexus::VmmRuntimeState +{ + fn from(s: types::VmmRuntimeState) -> Self { + Self { + state: s.state.into(), + gen: s.gen.into(), + time_updated: s.time_updated, + } + } +} + +impl From + for omicron_common::api::internal::nexus::SledInstanceState +{ + fn from(s: types::SledInstanceState) -> Self { + Self { + instance_state: s.instance_state.into(), + propolis_id: s.propolis_id, + vmm_state: s.vmm_state.into(), + } + } +} + impl From for omicron_common::api::external::InstanceState { diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 91ed7e4240..53512408af 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -739,6 +739,7 @@ pub enum ResourceType { UpdateableComponent, UserBuiltin, Zpool, + Vmm, } // IDENTITY METADATA @@ -866,25 +867,6 @@ impl InstanceState { InstanceState::Destroyed => "destroyed", } } - - /// Returns true if the given state represents a fully stopped Instance. - /// This means that a transition from an !is_stopped() state must go - /// through Stopping. - pub fn is_stopped(&self) -> bool { - match self { - InstanceState::Starting => false, - InstanceState::Running => false, - InstanceState::Stopping => false, - InstanceState::Rebooting => false, - InstanceState::Migrating => false, - - InstanceState::Creating => true, - InstanceState::Stopped => true, - InstanceState::Repairing => true, - InstanceState::Failed => true, - InstanceState::Destroyed => true, - } - } } /// The number of CPUs in an Instance @@ -912,17 +894,6 @@ pub struct InstanceRuntimeState { pub time_run_state_updated: DateTime, } -impl From - for InstanceRuntimeState -{ - fn from(state: crate::api::internal::nexus::InstanceRuntimeState) -> Self { - InstanceRuntimeState { - run_state: state.run_state, - time_run_state_updated: state.time_updated, - } - } -} - /// View of an Instance #[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct Instance { diff --git a/common/src/api/internal/nexus.rs b/common/src/api/internal/nexus.rs index 983976bbb7..a4a539ad9b 100644 --- a/common/src/api/internal/nexus.rs +++ b/common/src/api/internal/nexus.rs @@ -29,40 +29,59 @@ pub struct DiskRuntimeState { pub time_updated: DateTime, } -/// Runtime state of the Instance, including the actual running state and minimal -/// metadata -/// -/// This state is owned by the sled agent running that Instance. +/// The "static" properties of an instance: information about the instance that +/// doesn't change while the instance is running. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct InstanceRuntimeState { - /// runtime state of the Instance - pub run_state: InstanceState, - /// which sled is running this Instance - pub sled_id: Uuid, - /// which propolis-server is running this Instance - pub propolis_id: Uuid, - /// the target propolis-server during a migration of this Instance - pub dst_propolis_id: Option, - /// address of propolis-server running this Instance - pub propolis_addr: Option, - /// migration id (if one in process) - pub migration_id: Option, - /// The generation number for the Propolis and sled identifiers for this - /// instance. - pub propolis_gen: Generation, - /// number of CPUs allocated for this Instance +pub struct InstanceProperties { pub ncpus: InstanceCpuCount, - /// memory allocated for this Instance pub memory: ByteCount, - /// RFC1035-compliant hostname for the Instance. + /// RFC1035-compliant hostname for the instance. // TODO-cleanup different type? pub hostname: String, - /// generation number for this state +} + +/// The dynamic runtime properties of an instance: its current VMM ID (if any), +/// migration information (if any), and the instance state to report if there is +/// no active VMM. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InstanceRuntimeState { + /// The instance's currently active VMM ID. + pub propolis_id: Option, + /// If a migration is active, the ID of the target VMM. + pub dst_propolis_id: Option, + /// If a migration is active, the ID of that migration. + pub migration_id: Option, + /// Generation number for this state. pub gen: Generation, - /// timestamp for this information + /// Timestamp for this information. + pub time_updated: DateTime, +} + +/// The dynamic runtime properties of an individual VMM process. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct VmmRuntimeState { + /// The last state reported by this VMM. + pub state: InstanceState, + /// The generation number for this VMM's state. + pub gen: Generation, + /// Timestamp for the VMM's state. pub time_updated: DateTime, } +/// A wrapper type containing a sled's total knowledge of the state of a +/// specific VMM and the instance it incarnates. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct SledInstanceState { + /// The sled's conception of the state of the instance. + pub instance_state: InstanceRuntimeState, + + /// The ID of the VMM whose state is being reported. + pub propolis_id: Uuid, + + /// The most recent state of the sled's VMM process. + pub vmm_state: VmmRuntimeState, +} + // Oximeter producer/collector objects. /// Information announced by a metric server, used so that clients can contact it and collect diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 10e5546b6d..881b5831ba 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -26,7 +26,10 @@ use clap::Subcommand; use clap::ValueEnum; use diesel::expression::SelectableHelper; use diesel::query_dsl::QueryDsl; +use diesel::BoolExpressionMethods; use diesel::ExpressionMethods; +use diesel::JoinOnDsl; +use diesel::NullableExpressionMethods; use nexus_db_model::Dataset; use nexus_db_model::Disk; use nexus_db_model::DnsGroup; @@ -38,9 +41,11 @@ use nexus_db_model::Instance; use nexus_db_model::Project; use nexus_db_model::Region; use nexus_db_model::Sled; +use nexus_db_model::Vmm; use nexus_db_model::Zpool; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; +use nexus_db_queries::db::datastore::InstanceAndActiveVmm; use nexus_db_queries::db::identity::Asset; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::ServiceKind; @@ -61,6 +66,44 @@ use strum::IntoEnumIterator; use tabled::Tabled; use uuid::Uuid; +const NO_ACTIVE_PROPOLIS_MSG: &str = ""; +const NOT_ON_SLED_MSG: &str = ""; + +struct MaybePropolisId(Option); +struct MaybeSledId(Option); + +impl From<&InstanceAndActiveVmm> for MaybePropolisId { + fn from(value: &InstanceAndActiveVmm) -> Self { + Self(value.instance().runtime().propolis_id) + } +} + +impl Display for MaybePropolisId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(id) = self.0 { + write!(f, "{}", id) + } else { + write!(f, "{}", NO_ACTIVE_PROPOLIS_MSG) + } + } +} + +impl From<&InstanceAndActiveVmm> for MaybeSledId { + fn from(value: &InstanceAndActiveVmm) -> Self { + Self(value.sled_id()) + } +} + +impl Display for MaybeSledId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(id) = self.0 { + write!(f, "{}", id) + } else { + write!(f, "{}", NOT_ON_SLED_MSG) + } + } +} + #[derive(Debug, Args)] pub struct DbArgs { /// URL of the database SQL interface @@ -473,33 +516,54 @@ async fn cmd_db_disk_info( if let Some(instance_uuid) = disk.runtime().attach_instance_id { // Get the instance this disk is attached to use db::schema::instance::dsl as instance_dsl; - let instance = instance_dsl::instance + use db::schema::vmm::dsl as vmm_dsl; + let instances: Vec = instance_dsl::instance .filter(instance_dsl::id.eq(instance_uuid)) + .left_join( + vmm_dsl::vmm.on(vmm_dsl::id + .nullable() + .eq(instance_dsl::active_propolis_id) + .and(vmm_dsl::time_deleted.is_null())), + ) .limit(1) - .select(Instance::as_select()) + .select((Instance::as_select(), Option::::as_select())) .load_async(&*conn) .await - .context("loading requested instance")?; + .context("loading requested instance")? + .into_iter() + .map(|i: (Instance, Option)| i.into()) + .collect(); - let Some(instance) = instance.into_iter().next() else { + let Some(instance) = instances.into_iter().next() else { bail!("no instance: {} found", instance_uuid); }; - let instance_name = instance.name().to_string(); - let propolis_id = instance.runtime().propolis_id.to_string(); - let my_sled_id = instance.runtime().sled_id; + let instance_name = instance.instance().name().to_string(); + let disk_name = disk.name().to_string(); + let usr = if instance.vmm().is_some() { + let propolis_id = + instance.instance().runtime().propolis_id.unwrap(); + let my_sled_id = instance.sled_id().unwrap(); - let (_, my_sled) = LookupPath::new(opctx, datastore) - .sled_id(my_sled_id) - .fetch() - .await - .context("failed to look up sled")?; + let (_, my_sled) = LookupPath::new(opctx, datastore) + .sled_id(my_sled_id) + .fetch() + .await + .context("failed to look up sled")?; - let usr = UpstairsRow { - host_serial: my_sled.serial_number().to_string(), - disk_name: disk.name().to_string(), - instance_name, - propolis_zone: format!("oxz_propolis-server_{}", propolis_id), + UpstairsRow { + host_serial: my_sled.serial_number().to_string(), + disk_name, + instance_name, + propolis_zone: format!("oxz_propolis-server_{}", propolis_id), + } + } else { + UpstairsRow { + host_serial: NOT_ON_SLED_MSG.to_string(), + propolis_zone: NO_ACTIVE_PROPOLIS_MSG.to_string(), + disk_name, + instance_name, + } }; rows.push(usr); } else { @@ -691,7 +755,7 @@ async fn cmd_db_disk_physical( name: disk.name().to_string(), id: disk.id().to_string(), state: disk.runtime().disk_state, - instance_name: instance_name, + instance_name, }); } @@ -885,17 +949,17 @@ async fn cmd_db_sleds( struct CustomerInstanceRow { id: Uuid, state: String, - propolis_id: Uuid, - sled_id: Uuid, + propolis_id: MaybePropolisId, + sled_id: MaybeSledId, } -impl From for CustomerInstanceRow { - fn from(i: Instance) -> Self { +impl From for CustomerInstanceRow { + fn from(i: InstanceAndActiveVmm) -> Self { CustomerInstanceRow { - id: i.id(), - state: format!("{:?}", i.runtime_state.state.0), - propolis_id: i.runtime_state.propolis_id, - sled_id: i.runtime_state.sled_id, + id: i.instance().id(), + state: format!("{:?}", i.effective_state()), + propolis_id: (&i).into(), + sled_id: (&i).into(), } } } @@ -906,12 +970,22 @@ async fn cmd_db_instances( limit: NonZeroU32, ) -> Result<(), anyhow::Error> { use db::schema::instance::dsl; - let instances = dsl::instance + use db::schema::vmm::dsl as vmm_dsl; + let instances: Vec = dsl::instance + .left_join( + vmm_dsl::vmm.on(vmm_dsl::id + .nullable() + .eq(dsl::active_propolis_id) + .and(vmm_dsl::time_deleted.is_null())), + ) .limit(i64::from(u32::from(limit))) - .select(Instance::as_select()) + .select((Instance::as_select(), Option::::as_select())) .load_async(&*datastore.pool_connection_for_tests().await?) .await - .context("loading instances")?; + .context("loading instances")? + .into_iter() + .map(|i: (Instance, Option)| i.into()) + .collect(); let ctx = || "listing instances".to_string(); check_limit(&instances, limit, ctx); diff --git a/nexus/db-model/src/instance.rs b/nexus/db-model/src/instance.rs index d6aaa45de3..9252926547 100644 --- a/nexus/db-model/src/instance.rs +++ b/nexus/db-model/src/instance.rs @@ -8,18 +8,20 @@ use crate::schema::{disk, instance}; use chrono::{DateTime, Utc}; use db_macros::Resource; use nexus_types::external_api::params; -use nexus_types::identity::Resource; -use omicron_common::address::PROPOLIS_PORT; -use omicron_common::api::external; -use omicron_common::api::internal; use serde::Deserialize; use serde::Serialize; -use std::net::SocketAddr; use uuid::Uuid; /// An Instance (VM). #[derive( - Queryable, Insertable, Debug, Selectable, Resource, Serialize, Deserialize, + Clone, + Debug, + Queryable, + Insertable, + Selectable, + Resource, + Serialize, + Deserialize, )] #[diesel(table_name = instance)] pub struct Instance { @@ -32,25 +34,54 @@ pub struct Instance { /// user data for instance initialization systems (e.g. cloud-init) pub user_data: Vec, - /// runtime state of the Instance + /// The number of vCPUs (i.e., virtual logical processors) to allocate for + /// this instance. + #[diesel(column_name = ncpus)] + pub ncpus: InstanceCpuCount, + + /// The amount of guest memory to allocate for this instance. + #[diesel(column_name = memory)] + pub memory: ByteCount, + + /// The instance's hostname. + // TODO-cleanup: Different type? + #[diesel(column_name = hostname)] + pub hostname: String, + + #[diesel(column_name = boot_on_fault)] + pub boot_on_fault: bool, + #[diesel(embed)] pub runtime_state: InstanceRuntimeState, } impl Instance { + /// Constructs a new instance record with no VMM that will initially appear + /// to be in the Creating state. pub fn new( instance_id: Uuid, project_id: Uuid, params: ¶ms::InstanceCreate, - runtime: InstanceRuntimeState, ) -> Self { let identity = InstanceIdentity::new(instance_id, params.identity.clone()); + + let runtime_state = InstanceRuntimeState::new( + InstanceState::new( + omicron_common::api::external::InstanceState::Creating, + ), + identity.time_modified, + ); + Self { identity, project_id, user_data: params.user_data.clone(), - runtime_state: runtime, + ncpus: params.ncpus.into(), + memory: params.memory.into(), + hostname: params.hostname.clone(), + boot_on_fault: false, + runtime_state, } } @@ -59,20 +90,6 @@ impl Instance { } } -/// Conversion to the external API type. -impl Into for Instance { - fn into(self) -> external::Instance { - external::Instance { - identity: self.identity(), - project_id: self.project_id, - ncpus: self.runtime().ncpus.into(), - memory: self.runtime().memory.into(), - hostname: self.runtime().hostname.clone(), - runtime: self.runtime().clone().into(), - } - } -} - impl DatastoreAttachTargetConfig for Instance { type Id = Uuid; @@ -103,153 +120,95 @@ impl DatastoreAttachTargetConfig for Instance { // `diesel::prelude::AsChangeset`. #[diesel(table_name = instance, treat_none_as_null = true)] pub struct InstanceRuntimeState { - /// The instance's current user-visible instance state. + /// The instance state to fall back on if asked to compute this instance's + /// state while it has no active VMM. /// /// This field is guarded by the instance's `gen` field. #[diesel(column_name = state)] - pub state: InstanceState, + pub nexus_state: InstanceState, + /// The time at which the runtime state was last updated. This is distinct /// from the time the record was last modified, because some updates don't /// modify the runtime state. #[diesel(column_name = time_state_updated)] pub time_updated: DateTime, - /// The generation number for the instance's user-visible state. Each - /// successive state update from a single incarnation of an instance must - /// bear a new generation number. + + /// The generation number for the information stored in this structure, + /// including the fallback state, the instance's active Propolis ID, and its + /// migration IDs. #[diesel(column_name = state_generation)] pub gen: Generation, - /// The ID of the sled hosting the current incarnation of this instance. - /// - /// This field is guarded by the instance's `propolis_gen`. - // - // TODO(#2315): This should be optional so that it can be cleared when the - // instance is not active. - #[diesel(column_name = active_sled_id)] - pub sled_id: Uuid, + /// The ID of the Propolis server hosting the current incarnation of this - /// instance. + /// instance, or None if the instance has no active VMM. /// - /// This field is guarded by the instance's `propolis_gen`. + /// This field is guarded by the instance's `gen`. #[diesel(column_name = active_propolis_id)] - pub propolis_id: Uuid, - /// The IP of the instance's current Propolis server. - /// - /// This field is guarded by the instance's `propolis_gen`. - #[diesel(column_name = active_propolis_ip)] - pub propolis_ip: Option, + pub propolis_id: Option, + /// If a migration is in progress, the ID of the Propolis server that is - /// the migration target. Note that the target's sled agent will have a - /// runtime state where `propolis_id` and `dst_propolis_id` are equal. + /// the migration target. /// - /// This field is guarded by the instance's `propolis_gen`. + /// This field is guarded by the instance's `gen`. #[diesel(column_name = target_propolis_id)] pub dst_propolis_id: Option, + /// If a migration is in progress, a UUID identifying that migration. This /// can be used to provide mutual exclusion between multiple attempts to /// migrate and between an attempt to migrate an attempt to mutate an /// instance in a way that's incompatible with migration. /// - /// This field is guarded by the instance's `propolis_gen`. + /// This field is guarded by the instance's `gen`. #[diesel(column_name = migration_id)] pub migration_id: Option, - /// A generation number protecting the instance's "location" information: - /// its sled ID, Propolis ID and IP, and migration information. Each state - /// update that updates one or more of these fields must bear a new - /// Propolis generation. - /// - /// Records with new Propolis generations supersede records with older - /// generations irrespective of their state generations. That is, a record - /// with Propolis generation 4 and state generation 1 is "newer" than - /// a record with Propolis generation 3 and state generation 5. - #[diesel(column_name = propolis_generation)] - pub propolis_gen: Generation, - /// The number of vCPUs (i.e., virtual logical processors) to allocate for - /// this instance. - #[diesel(column_name = ncpus)] - pub ncpus: InstanceCpuCount, - /// The amount of guest memory to allocate for this instance. - #[diesel(column_name = memory)] - pub memory: ByteCount, - /// The instance's hostname. - // TODO-cleanup: Different type? - #[diesel(column_name = hostname)] - pub hostname: String, - #[diesel(column_name = boot_on_fault)] - pub boot_on_fault: bool, } -impl From - for sled_agent_client::types::InstanceRuntimeState -{ - fn from(s: InstanceRuntimeState) -> Self { +impl InstanceRuntimeState { + fn new(initial_state: InstanceState, creation_time: DateTime) -> Self { Self { - run_state: s.state.into(), - sled_id: s.sled_id, - propolis_id: s.propolis_id, - dst_propolis_id: s.dst_propolis_id, - propolis_addr: s - .propolis_ip - .map(|ip| SocketAddr::new(ip.ip(), PROPOLIS_PORT).to_string()), - migration_id: s.migration_id, - propolis_gen: s.propolis_gen.into(), - ncpus: s.ncpus.into(), - memory: s.memory.into(), - hostname: s.hostname, - gen: s.gen.into(), - time_updated: s.time_updated, + nexus_state: initial_state, + time_updated: creation_time, + propolis_id: None, + dst_propolis_id: None, + migration_id: None, + gen: Generation::new(), } } } -/// Conversion to the external API type. -impl Into for InstanceRuntimeState { - fn into(self) -> external::InstanceRuntimeState { - external::InstanceRuntimeState { - run_state: *self.state.state(), - time_run_state_updated: self.time_updated, - } - } -} +impl From + for InstanceRuntimeState +{ + fn from( + state: omicron_common::api::internal::nexus::InstanceRuntimeState, + ) -> Self { + let nexus_state = if state.propolis_id.is_some() { + omicron_common::api::external::InstanceState::Running + } else { + omicron_common::api::external::InstanceState::Stopped + }; -/// Conversion from the internal API type. -impl From for InstanceRuntimeState { - fn from(state: internal::nexus::InstanceRuntimeState) -> Self { Self { - state: InstanceState::new(state.run_state), - sled_id: state.sled_id, + nexus_state: InstanceState::new(nexus_state), + time_updated: state.time_updated, + gen: state.gen.into(), propolis_id: state.propolis_id, dst_propolis_id: state.dst_propolis_id, - propolis_ip: state.propolis_addr.map(|addr| addr.ip().into()), migration_id: state.migration_id, - propolis_gen: state.propolis_gen.into(), - ncpus: state.ncpus.into(), - memory: state.memory.into(), - hostname: state.hostname, - gen: state.gen.into(), - time_updated: state.time_updated, - boot_on_fault: false, } } } -/// Conversion to the internal API type. -impl Into for InstanceRuntimeState { - fn into(self) -> internal::nexus::InstanceRuntimeState { - internal::nexus::InstanceRuntimeState { - run_state: *self.state.state(), - sled_id: self.sled_id, - propolis_id: self.propolis_id, - dst_propolis_id: self.dst_propolis_id, - propolis_addr: self - .propolis_ip - .map(|ip| SocketAddr::new(ip.ip(), PROPOLIS_PORT)), - propolis_gen: self.propolis_gen.into(), - migration_id: self.migration_id, - ncpus: self.ncpus.into(), - memory: self.memory.into(), - hostname: self.hostname, - gen: self.gen.into(), - time_updated: self.time_updated, +impl From + for sled_agent_client::types::InstanceRuntimeState +{ + fn from(state: InstanceRuntimeState) -> Self { + Self { + dst_propolis_id: state.dst_propolis_id, + gen: state.gen.into(), + migration_id: state.migration_id, + propolis_id: state.propolis_id, + time_updated: state.time_updated, } } } diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 334dedad9f..f1447fc503 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -77,6 +77,7 @@ mod update_artifact; mod user_builtin; mod virtual_provisioning_collection; mod virtual_provisioning_resource; +mod vmm; mod vni; mod volume; mod vpc; @@ -156,6 +157,7 @@ pub use update_artifact::*; pub use user_builtin::*; pub use virtual_provisioning_collection::*; pub use virtual_provisioning_resource::*; +pub use vmm::*; pub use vni::*; pub use volume::*; pub use vpc::*; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 0165ab1568..2d6970452d 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -344,19 +344,30 @@ table! { time_deleted -> Nullable, project_id -> Uuid, user_data -> Binary, + ncpus -> Int8, + memory -> Int8, + hostname -> Text, + boot_on_fault -> Bool, state -> crate::InstanceStateEnum, time_state_updated -> Timestamptz, state_generation -> Int8, - active_sled_id -> Uuid, - active_propolis_id -> Uuid, - active_propolis_ip -> Nullable, + active_propolis_id -> Nullable, target_propolis_id -> Nullable, migration_id -> Nullable, - propolis_generation -> Int8, - ncpus -> Int8, - memory -> Int8, - hostname -> Text, - boot_on_fault -> Bool, + } +} + +table! { + vmm (id) { + id -> Uuid, + time_created -> Timestamptz, + time_deleted -> Nullable, + instance_id -> Uuid, + sled_id -> Uuid, + propolis_ip -> Inet, + state -> crate::InstanceStateEnum, + time_state_updated -> Timestamptz, + state_generation -> Int8, } } @@ -1168,6 +1179,7 @@ allow_tables_to_appear_in_same_query!( sled, sled_resource, router_route, + vmm, volume, vpc, vpc_subnet, diff --git a/nexus/db-model/src/vmm.rs b/nexus/db-model/src/vmm.rs new file mode 100644 index 0000000000..fe1158d5bb --- /dev/null +++ b/nexus/db-model/src/vmm.rs @@ -0,0 +1,137 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Defines database model types for the Vmm table. +//! +//! A row in the Vmm table stores information about a single Propolis VMM +//! running on a specific sled that incarnates a specific instance. A VMM's +//! instance ID, sled assignment, and Propolis server IP are all fixed for the +//! lifetime of the VMM. As with instances, the VMM's lifecycle-related state is +//! broken out into a separate type that allows sled agent and Nexus to send VMM +//! state updates to each other without sending parameters that are useless to +//! sled agent or that sled agent will never update (like the sled ID). + +use super::{Generation, InstanceState}; +use crate::schema::vmm; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// An individual VMM process that incarnates a specific instance. +#[derive( + Clone, Queryable, Debug, Selectable, Serialize, Deserialize, Insertable, +)] +#[diesel(table_name = vmm)] +pub struct Vmm { + /// This VMM's primary ID, referred to by an `Instance`'s `propolis_id` or + /// `target_propolis_id` fields. + pub id: Uuid, + + /// The time this VMM record was created. + pub time_created: DateTime, + + /// The time this VMM was destroyed. + pub time_deleted: Option>, + + /// The ID of the `Instance` that owns this VMM. + pub instance_id: Uuid, + + /// The sled assigned to the care and feeding of this VMM. + pub sled_id: Uuid, + + /// The IP address at which this VMM is serving the Propolis server API. + pub propolis_ip: ipnetwork::IpNetwork, + + /// Runtime state for the VMM. + #[diesel(embed)] + pub runtime: VmmRuntimeState, +} + +/// The set of states that a VMM can have when it is created. +pub enum VmmInitialState { + Starting, + Migrating, +} + +impl Vmm { + /// Creates a new VMM record. + pub fn new( + id: Uuid, + instance_id: Uuid, + sled_id: Uuid, + propolis_ip: ipnetwork::IpNetwork, + initial_state: VmmInitialState, + ) -> Self { + use omicron_common::api::external::InstanceState as ApiInstanceState; + + let now = Utc::now(); + let api_state = match initial_state { + VmmInitialState::Starting => ApiInstanceState::Starting, + VmmInitialState::Migrating => ApiInstanceState::Migrating, + }; + + Self { + id, + time_created: now, + time_deleted: None, + instance_id, + sled_id, + propolis_ip, + runtime: VmmRuntimeState { + state: InstanceState::new(api_state), + time_state_updated: now, + gen: Generation::new(), + }, + } + } +} + +/// Runtime state for a VMM, owned by the sled where that VMM is running. +#[derive( + Clone, + Debug, + AsChangeset, + Selectable, + Insertable, + Queryable, + Serialize, + Deserialize, +)] +#[diesel(table_name = vmm)] +pub struct VmmRuntimeState { + /// The state of this VMM. If this VMM is the active VMM for a given + /// instance, this state is the instance's logical state. + pub state: InstanceState, + + /// The time at which this state was most recently updated. + pub time_state_updated: DateTime, + + /// The generation number protecting this VMM's state and update time. + #[diesel(column_name = state_generation)] + pub gen: Generation, +} + +impl From + for VmmRuntimeState +{ + fn from( + value: omicron_common::api::internal::nexus::VmmRuntimeState, + ) -> Self { + Self { + state: InstanceState::new(value.state), + time_state_updated: value.time_updated, + gen: value.gen.into(), + } + } +} + +impl From for sled_agent_client::types::VmmRuntimeState { + fn from(s: Vmm) -> Self { + Self { + gen: s.runtime.gen.into(), + state: s.runtime.state.into(), + time_updated: s.runtime.time_state_updated, + } + } +} diff --git a/nexus/db-queries/src/db/datastore/disk.rs b/nexus/db-queries/src/db/datastore/disk.rs index 80f72c1e18..a0d9bf12c3 100644 --- a/nexus/db-queries/src/db/datastore/disk.rs +++ b/nexus/db-queries/src/db/datastore/disk.rs @@ -190,7 +190,9 @@ impl DataStore { authz_instance.id(), authz_disk.id(), instance::table.into_boxed().filter( - instance::dsl::state.eq_any(ok_to_attach_instance_states), + instance::dsl::state + .eq_any(ok_to_attach_instance_states) + .and(instance::dsl::active_propolis_id.is_null()), ), disk::table.into_boxed().filter( disk::dsl::disk_state.eq_any(ok_to_attach_disk_state_labels), @@ -230,7 +232,15 @@ impl DataStore { // why we did not attach. api::external::DiskState::Creating | api::external::DiskState::Detached => { - match collection.runtime_state.state.state() { + if collection.runtime_state.propolis_id.is_some() { + return Err( + Error::invalid_request( + "cannot attach disk: instance is not \ + fully stopped" + ) + ); + } + match collection.runtime_state.nexus_state.state() { // Ok-to-be-attached instance states: api::external::InstanceState::Creating | api::external::InstanceState::Stopped => { @@ -254,7 +264,7 @@ impl DataStore { _ => { Err(Error::invalid_request(&format!( "cannot attach disk to instance in {} state", - collection.runtime_state.state.state(), + collection.runtime_state.nexus_state.state(), ))) } } @@ -320,7 +330,9 @@ impl DataStore { authz_disk.id(), instance::table .into_boxed() - .filter(instance::dsl::state.eq_any(ok_to_detach_instance_states)), + .filter(instance::dsl::state + .eq_any(ok_to_detach_instance_states) + .and(instance::dsl::active_propolis_id.is_null())), disk::table .into_boxed() .filter(disk::dsl::disk_state.eq_any(ok_to_detach_disk_state_labels)), @@ -361,7 +373,15 @@ impl DataStore { // Ok-to-detach disk states: Inspect the state to infer // why we did not detach. api::external::DiskState::Attached(id) if id == authz_instance.id() => { - match collection.runtime_state.state.state() { + if collection.runtime_state.propolis_id.is_some() { + return Err( + Error::invalid_request( + "cannot attach disk: instance is not \ + fully stopped" + ) + ); + } + match collection.runtime_state.nexus_state.state() { // Ok-to-be-detached instance states: api::external::InstanceState::Creating | api::external::InstanceState::Stopped => { @@ -375,7 +395,7 @@ impl DataStore { _ => { Err(Error::invalid_request(&format!( "cannot detach disk from instance in {} state", - collection.runtime_state.state.state(), + collection.runtime_state.nexus_state.state(), ))) } } diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index 46ca07a74a..188f5c30c9 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -21,12 +21,14 @@ use crate::db::model::Instance; use crate::db::model::InstanceRuntimeState; use crate::db::model::Name; use crate::db::model::Project; +use crate::db::model::Vmm; use crate::db::pagination::paginated; use crate::db::update_and_check::UpdateAndCheck; use crate::db::update_and_check::UpdateStatus; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; +use nexus_db_model::VmmRuntimeState; use omicron_common::api; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; @@ -40,6 +42,68 @@ use omicron_common::bail_unless; use ref_cast::RefCast; use uuid::Uuid; +/// Wraps a record of an `Instance` along with its active `Vmm`, if it has one. +#[derive(Clone, Debug)] +pub struct InstanceAndActiveVmm { + instance: Instance, + vmm: Option, +} + +impl InstanceAndActiveVmm { + pub fn instance(&self) -> &Instance { + &self.instance + } + + pub fn vmm(&self) -> &Option { + &self.vmm + } + + pub fn sled_id(&self) -> Option { + self.vmm.as_ref().map(|v| v.sled_id) + } + + pub fn effective_state( + &self, + ) -> omicron_common::api::external::InstanceState { + if let Some(vmm) = &self.vmm { + vmm.runtime.state.0 + } else { + self.instance.runtime().nexus_state.0 + } + } +} + +impl From<(Instance, Option)> for InstanceAndActiveVmm { + fn from(value: (Instance, Option)) -> Self { + Self { instance: value.0, vmm: value.1 } + } +} + +impl From for omicron_common::api::external::Instance { + fn from(value: InstanceAndActiveVmm) -> Self { + let (run_state, time_run_state_updated) = if let Some(vmm) = value.vmm { + (vmm.runtime.state, vmm.runtime.time_state_updated) + } else { + ( + value.instance.runtime_state.nexus_state.clone(), + value.instance.runtime_state.time_updated, + ) + }; + + Self { + identity: value.instance.identity(), + project_id: value.instance.project_id, + ncpus: value.instance.ncpus.into(), + memory: value.instance.memory.into(), + hostname: value.instance.hostname, + runtime: omicron_common::api::external::InstanceRuntimeState { + run_state: *run_state.state(), + time_run_state_updated, + }, + } + } +} + impl DataStore { /// Idempotently insert a database record for an Instance /// @@ -97,10 +161,10 @@ impl DataStore { })?; bail_unless!( - instance.runtime().state.state() + instance.runtime().nexus_state.state() == &api::external::InstanceState::Creating, "newly-created Instance has unexpected state: {:?}", - instance.runtime().state + instance.runtime().nexus_state ); bail_unless!( instance.runtime().gen == gen, @@ -115,11 +179,12 @@ impl DataStore { opctx: &OpContext, authz_project: &authz::Project, pagparams: &PaginatedBy<'_>, - ) -> ListResultVec { + ) -> ListResultVec { opctx.authorize(authz::Action::ListChildren, authz_project).await?; use db::schema::instance::dsl; - match pagparams { + use db::schema::vmm::dsl as vmm_dsl; + Ok(match pagparams { PaginatedBy::Id(pagparams) => { paginated(dsl::instance, dsl::id, &pagparams) } @@ -131,10 +196,21 @@ impl DataStore { } .filter(dsl::project_id.eq(authz_project.id())) .filter(dsl::time_deleted.is_null()) - .select(Instance::as_select()) - .load_async::(&*self.pool_connection_authorized(opctx).await?) + .left_join( + vmm_dsl::vmm.on(vmm_dsl::id + .nullable() + .eq(dsl::active_propolis_id) + .and(vmm_dsl::time_deleted.is_null())), + ) + .select((Instance::as_select(), Option::::as_select())) + .load_async::<(Instance, Option)>( + &*self.pool_connection_authorized(opctx).await?, + ) .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|(instance, vmm)| InstanceAndActiveVmm { instance, vmm }) + .collect()) } /// Fetches information about an Instance that the caller has previously @@ -160,22 +236,29 @@ impl DataStore { Ok(db_instance) } - /// Fetches information about a deleted instance. This can be used to - /// query the properties an instance had at the time it was deleted, which - /// can be useful when cleaning up a deleted instance. - pub async fn instance_fetch_deleted( + pub async fn instance_fetch_with_vmm( &self, opctx: &OpContext, authz_instance: &authz::Instance, - ) -> LookupResult { + ) -> LookupResult { opctx.authorize(authz::Action::Read, authz_instance).await?; - use db::schema::instance::dsl; - let instance = dsl::instance - .filter(dsl::id.eq(authz_instance.id())) - .filter(dsl::time_deleted.is_not_null()) - .select(Instance::as_select()) - .get_result_async(&*self.pool_connection_authorized(opctx).await?) + use db::schema::instance::dsl as instance_dsl; + use db::schema::vmm::dsl as vmm_dsl; + + let (instance, vmm) = instance_dsl::instance + .filter(instance_dsl::id.eq(authz_instance.id())) + .filter(instance_dsl::time_deleted.is_null()) + .left_join( + vmm_dsl::vmm.on(vmm_dsl::id + .nullable() + .eq(instance_dsl::active_propolis_id) + .and(vmm_dsl::time_deleted.is_null())), + ) + .select((Instance::as_select(), Option::::as_select())) + .get_result_async::<(Instance, Option)>( + &*self.pool_connection_authorized(opctx).await?, + ) .await .map_err(|e| { public_error_from_diesel( @@ -187,7 +270,7 @@ impl DataStore { ) })?; - Ok(instance) + Ok(InstanceAndActiveVmm { instance, vmm }) } // TODO-design It's tempting to return the updated state of the Instance @@ -211,15 +294,7 @@ impl DataStore { // - the active Propolis ID will not change, the state generation // increased, and the Propolis generation will not change, or // - the Propolis generation increased. - .filter( - (dsl::active_propolis_id - .eq(new_runtime.propolis_id) - .and(dsl::state_generation.lt(new_runtime.gen)) - .and( - dsl::propolis_generation.eq(new_runtime.propolis_gen), - )) - .or(dsl::propolis_generation.lt(new_runtime.propolis_gen)), - ) + .filter(dsl::state_generation.lt(new_runtime.gen)) .set(new_runtime.clone()) .check_if_exists::(*instance_id) .execute_and_check(&*self.pool_connection_unauthorized().await?) @@ -241,6 +316,69 @@ impl DataStore { Ok(updated) } + /// Updates an instance record and a VMM record with a single database + /// command. + /// + /// This is intended to be used to apply updates from sled agent that + /// may change a VMM's runtime state (e.g. moving an instance from Running + /// to Stopped) and its corresponding instance's state (e.g. changing the + /// active Propolis ID to reflect a completed migration) in a single + /// transaction. The caller is responsible for ensuring the instance and + /// VMM states are consistent with each other before calling this routine. + /// + /// # Arguments + /// + /// - instance_id: The ID of the instance to update. + /// - new_instance: The new instance runtime state to try to write. + /// - vmm_id: The ID of the VMM to update. + /// - new_vmm: The new VMM runtime state to try to write. + /// + /// # Return value + /// + /// - `Ok((instance_updated, vmm_updated))` if the query was issued + /// successfully. `instance_updated` and `vmm_updated` are each true if + /// the relevant item was updated and false otherwise. Note that an update + /// can fail because it was inapplicable (i.e. the database has state with + /// a newer generation already) or because the relevant record was not + /// found. + /// - `Err` if another error occurred while accessing the database. + pub async fn instance_and_vmm_update_runtime( + &self, + instance_id: &Uuid, + new_instance: &InstanceRuntimeState, + vmm_id: &Uuid, + new_vmm: &VmmRuntimeState, + ) -> Result<(bool, bool), Error> { + let query = crate::db::queries::instance::InstanceAndVmmUpdate::new( + *instance_id, + new_instance.clone(), + *vmm_id, + new_vmm.clone(), + ); + + // The InstanceAndVmmUpdate query handles and indicates failure to find + // either the instance or the VMM, so a query failure here indicates + // some kind of internal error and not a failed lookup. + let result = query + .execute_and_check(&*self.pool_connection_unauthorized().await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + let instance_updated = match result.instance_status { + Some(UpdateStatus::Updated) => true, + Some(UpdateStatus::NotUpdatedButExists) => false, + None => false, + }; + + let vmm_updated = match result.vmm_status { + Some(UpdateStatus::Updated) => true, + Some(UpdateStatus::NotUpdatedButExists) => false, + None => false, + }; + + Ok((instance_updated, vmm_updated)) + } + pub async fn project_delete_instance( &self, opctx: &OpContext, @@ -270,7 +408,9 @@ impl DataStore { let _instance = Instance::detach_resources( authz_instance.id(), instance::table.into_boxed().filter( - instance::dsl::state.eq_any(ok_to_delete_instance_states), + instance::dsl::state + .eq_any(ok_to_delete_instance_states) + .and(instance::dsl::active_propolis_id.is_null()), ), disk::table.into_boxed().filter( disk::dsl::disk_state.eq_any(ok_to_detach_disk_state_labels), @@ -295,7 +435,14 @@ impl DataStore { &authz_instance.id(), ), DetachManyError::NoUpdate { collection } => { - let instance_state = collection.runtime_state.state.state(); + if collection.runtime_state.propolis_id.is_some() { + return Error::invalid_request( + "cannot delete instance: instance is running or has \ + not yet fully stopped", + ); + } + let instance_state = + collection.runtime_state.nexus_state.state(); match instance_state { api::external::InstanceState::Stopped | api::external::InstanceState::Failed => { diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 7d5e32cad9..a77e20647a 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -82,6 +82,7 @@ mod switch_interface; mod switch_port; mod update; mod virtual_provisioning_collection; +mod vmm; mod volume; mod vpc; mod zpool; @@ -91,6 +92,7 @@ pub use db_metadata::{ all_sql_for_version_migration, EARLIEST_SUPPORTED_VERSION, }; pub use dns::DnsVersionUpdateBuilder; +pub use instance::InstanceAndActiveVmm; pub use rack::RackInit; pub use silo::Discoverability; pub use switch_port::SwitchPortSettingsCombinedResult; diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 4a46b23529..06550e9439 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -471,12 +471,11 @@ impl DataStore { let conn = self.pool_connection_authorized(opctx).await?; if primary { conn.transaction_async(|conn| async move { - let instance_state = instance_query - .get_result_async(&conn) - .await? - .runtime_state - .state; - if instance_state != stopped { + let instance_runtime = + instance_query.get_result_async(&conn).await?.runtime_state; + if instance_runtime.propolis_id.is_some() + || instance_runtime.nexus_state != stopped + { return Err(TxnError::CustomError( NetworkInterfaceUpdateError::InstanceNotStopped, )); @@ -515,12 +514,11 @@ impl DataStore { // we're only hitting a single row. Note that we still need to // verify the instance is stopped. conn.transaction_async(|conn| async move { - let instance_state = instance_query - .get_result_async(&conn) - .await? - .runtime_state - .state; - if instance_state != stopped { + let instance_state = + instance_query.get_result_async(&conn).await?.runtime_state; + if instance_state.propolis_id.is_some() + || instance_state.nexus_state != stopped + { return Err(TxnError::CustomError( NetworkInterfaceUpdateError::InstanceNotStopped, )); diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index a52d1b7772..f4f5188057 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -115,6 +115,7 @@ impl DataStore { resource_dsl::hardware_threads::NAME )) + resources.hardware_threads) .le(sled_dsl::usable_hardware_threads); + // This answers the boolean question: // "Does the SUM of all RAM usage, plus the one we're trying // to allocate, consume less RAM than exists on the sled?" @@ -125,6 +126,15 @@ impl DataStore { )) + resources.rss_ram) .le(sled_dsl::usable_physical_ram); + // Determine whether adding this service's reservoir allocation + // to what's allocated on the sled would avoid going over quota. + let sled_has_space_in_reservoir = + (diesel::dsl::sql::(&format!( + "COALESCE(SUM(CAST({} as INT8)), 0)", + resource_dsl::reservoir_ram::NAME + )) + resources.reservoir_ram) + .le(sled_dsl::reservoir_size); + // Generate a query describing all of the sleds that have space // for this reservation. let mut sled_targets = sled_dsl::sled @@ -134,8 +144,9 @@ impl DataStore { ) .group_by(sled_dsl::id) .having( - sled_has_space_for_threads.and(sled_has_space_for_rss), - // TODO: We should also validate the reservoir space, when it exists. + sled_has_space_for_threads + .and(sled_has_space_for_rss) + .and(sled_has_space_in_reservoir), ) .filter(sled_dsl::time_deleted.is_null()) .select(sled_dsl::id) diff --git a/nexus/db-queries/src/db/datastore/vmm.rs b/nexus/db-queries/src/db/datastore/vmm.rs new file mode 100644 index 0000000000..18afde84f0 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/vmm.rs @@ -0,0 +1,161 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! [`DataStore`] helpers for working with VMM records. + +use super::DataStore; +use crate::authz; +use crate::context::OpContext; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::model::Vmm; +use crate::db::model::VmmRuntimeState; +use crate::db::schema::vmm::dsl; +use crate::db::update_and_check::UpdateAndCheck; +use crate::db::update_and_check::UpdateStatus; +use async_bb8_diesel::AsyncRunQueryDsl; +use chrono::Utc; +use diesel::prelude::*; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::LookupResult; +use omicron_common::api::external::LookupType; +use omicron_common::api::external::ResourceType; +use omicron_common::api::external::UpdateResult; +use uuid::Uuid; + +impl DataStore { + pub async fn vmm_insert( + &self, + opctx: &OpContext, + vmm: Vmm, + ) -> CreateResult { + let vmm = diesel::insert_into(dsl::vmm) + .values(vmm) + .on_conflict(dsl::id) + .do_update() + .set(dsl::time_state_updated.eq(dsl::time_state_updated)) + .returning(Vmm::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(vmm) + } + + pub async fn vmm_mark_deleted( + &self, + opctx: &OpContext, + vmm_id: &Uuid, + ) -> UpdateResult { + use crate::db::model::InstanceState as DbInstanceState; + use omicron_common::api::external::InstanceState as ApiInstanceState; + + let valid_states = vec![ + DbInstanceState::new(ApiInstanceState::Destroyed), + DbInstanceState::new(ApiInstanceState::Failed), + ]; + + let updated = diesel::update(dsl::vmm) + .filter(dsl::id.eq(*vmm_id)) + .filter(dsl::state.eq_any(valid_states)) + .set(dsl::time_deleted.eq(Utc::now())) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Vmm, + LookupType::ById(*vmm_id), + ), + ) + })?; + + Ok(updated != 0) + } + + pub async fn vmm_fetch( + &self, + opctx: &OpContext, + authz_instance: &authz::Instance, + vmm_id: &Uuid, + ) -> LookupResult { + opctx.authorize(authz::Action::Read, authz_instance).await?; + + let vmm = dsl::vmm + .filter(dsl::id.eq(*vmm_id)) + .filter(dsl::instance_id.eq(authz_instance.id())) + .filter(dsl::time_deleted.is_null()) + .select(Vmm::as_select()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Vmm, + LookupType::ById(*vmm_id), + ), + ) + })?; + + Ok(vmm) + } + + pub async fn vmm_update_runtime( + &self, + vmm_id: &Uuid, + new_runtime: &VmmRuntimeState, + ) -> Result { + let updated = diesel::update(dsl::vmm) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(*vmm_id)) + .filter(dsl::state_generation.lt(new_runtime.gen)) + .set(new_runtime.clone()) + .check_if_exists::(*vmm_id) + .execute_and_check(&*self.pool_connection_unauthorized().await?) + .await + .map(|r| match r.status { + UpdateStatus::Updated => true, + UpdateStatus::NotUpdatedButExists => false, + }) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Vmm, + LookupType::ById(*vmm_id), + ), + ) + })?; + + Ok(updated) + } + + /// Forcibly overwrites the Propolis IP in the supplied VMM's record with + /// the supplied Propolis IP. + /// + /// This is used in tests to overwrite the IP for a VMM that is backed by a + /// mock Propolis server that serves on localhost but has its Propolis IP + /// allocated by the instance start procedure. (Unfortunately, this can't be + /// marked #[cfg(test)] because the integration tests require this + /// functionality.) + pub async fn vmm_overwrite_ip_for_test( + &self, + opctx: &OpContext, + vmm_id: &Uuid, + new_ip: ipnetwork::IpNetwork, + ) -> UpdateResult { + let vmm = diesel::update(dsl::vmm) + .filter(dsl::id.eq(*vmm_id)) + .set(dsl::propolis_ip.eq(new_ip)) + .returning(Vmm::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(vmm) + } +} diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 46c3d2504e..14886ba018 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -572,7 +572,7 @@ impl DataStore { // Sleds to notify when firewall rules change. use db::schema::{ instance, instance_network_interface, service, - service_network_interface, sled, + service_network_interface, sled, vmm, }; let instance_query = instance_network_interface::table @@ -581,10 +581,15 @@ impl DataStore { .on(instance::id .eq(instance_network_interface::instance_id)), ) - .inner_join(sled::table.on(sled::id.eq(instance::active_sled_id))) + .inner_join( + vmm::table + .on(vmm::id.nullable().eq(instance::active_propolis_id)), + ) + .inner_join(sled::table.on(sled::id.eq(vmm::sled_id))) .filter(instance_network_interface::vpc_id.eq(vpc_id)) .filter(instance_network_interface::time_deleted.is_null()) .filter(instance::time_deleted.is_null()) + .filter(vmm::time_deleted.is_null()) .select(Sled::as_select()); let service_query = service_network_interface::table diff --git a/nexus/db-queries/src/db/queries/instance.rs b/nexus/db-queries/src/db/queries/instance.rs new file mode 100644 index 0000000000..ea40877450 --- /dev/null +++ b/nexus/db-queries/src/db/queries/instance.rs @@ -0,0 +1,255 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Implement a query for updating an instance and VMM in a single CTE. + +use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::prelude::QueryResult; +use diesel::query_builder::{Query, QueryFragment, QueryId}; +use diesel::result::Error as DieselError; +use diesel::sql_types::{Nullable, Uuid as SqlUuid}; +use diesel::{pg::Pg, query_builder::AstPass}; +use diesel::{Column, ExpressionMethods, QueryDsl, RunQueryDsl}; +use nexus_db_model::{ + schema::{instance::dsl as instance_dsl, vmm::dsl as vmm_dsl}, + InstanceRuntimeState, VmmRuntimeState, +}; +use uuid::Uuid; + +use crate::db::pool::DbConnection; +use crate::db::update_and_check::UpdateStatus; + +/// A CTE that checks and updates the instance and VMM tables in a single +/// atomic operation. +// +// The single-table update-and-check CTE has the following form: +// +// WITH found AS (SELECT FROM T WHERE ) +// updated AS (UPDATE T SET RETURNING *) +// SELECT +// found. +// updated. +// found.* +// FROM +// found +// LEFT JOIN +// updated +// ON +// found. = updated.; +// +// The idea behind this query is to have separate "found" and "updated" +// subqueries for the instance and VMM tables, then use those to create two more +// subqueries that perform the joins and yield the results, along the following +// lines: +// +// WITH vmm_found AS (SELECT(SELECT id FROM vmm WHERE vmm.id = id) AS id), +// vmm_updated AS (UPDATE vmm SET ... RETURNING *), +// instance_found AS (SELECT( +// SELECT id FROM instance WHERE instance.id = id +// ) AS id), +// instance_updated AS (UPDATE instance SET ... RETURNING *), +// vmm_result AS ( +// SELECT vmm_found.id AS found, vmm_updated.id AS updated +// FROM vmm_found +// LEFT JOIN vmm_updated +// ON vmm_found.id = vmm_updated.id +// ), +// instance_result AS ( +// SELECT instance_found.id AS found, instance_updated.id AS updated +// FROM instance_found +// LEFT JOIN instance_updated +// ON instance_found.id = instance_updated.id +// ) +// SELECT vmm_result.found, vmm_result.updated, instance_result.found, +// instance_result.updated +// FROM vmm_result, instance_result; +// +// The "wrapper" SELECTs when finding instances and VMMs are used to get a NULL +// result in the final output instead of failing the entire query if the target +// object is missing. This maximizes Nexus's flexibility when dealing with +// updates from sled agent that refer to one valid and one deleted object. (This +// can happen if, e.g., sled agent sends a message indicating that a retired VMM +// has finally been destroyed when its instance has since been deleted.) +pub struct InstanceAndVmmUpdate { + instance_find: Box + Send>, + vmm_find: Box + Send>, + instance_update: Box + Send>, + vmm_update: Box + Send>, +} + +/// Contains the result of a combined instance-and-VMM update operation. +#[derive(Copy, Clone, PartialEq, Debug)] +pub struct InstanceAndVmmUpdateResult { + /// `Some(status)` if the target instance was found; the wrapped + /// `UpdateStatus` indicates whether the row was updated. `None` if the + /// instance was not found. + pub instance_status: Option, + + /// `Some(status)` if the target VMM was found; the wrapped `UpdateStatus` + /// indicates whether the row was updated. `None` if the VMM was not found. + pub vmm_status: Option, +} + +/// Computes the update status to return from the results of queries that find +/// and update an object with an ID of type `T`. +fn compute_update_status( + found: Option, + updated: Option, +) -> Option +where + T: PartialEq + std::fmt::Display, +{ + match (found, updated) { + // If both the "find" and "update" prongs returned an ID, the row was + // updated. The IDs should match in this case (if they don't then the + // query was constructed very strangely!). + (Some(found_id), Some(updated_id)) if found_id == updated_id => { + Some(UpdateStatus::Updated) + } + // If the "find" prong returned an ID but the "update" prong didn't, the + // row exists but wasn't updated. + (Some(_), None) => Some(UpdateStatus::NotUpdatedButExists), + // If neither prong returned anything, indicate the row is missing. + (None, None) => None, + // If both prongs returned an ID, but they don't match, something + // terrible has happened--the prongs must have referred to different + // IDs! + (Some(found_id), Some(mismatched_id)) => unreachable!( + "updated ID {} didn't match found ID {}", + mismatched_id, found_id + ), + // Similarly, if the target ID was not found but something was updated + // anyway, then something is wrong with the update query--either it has + // the wrong ID or did not filter rows properly. + (None, Some(updated_id)) => unreachable!( + "ID {} was updated but no found ID was supplied", + updated_id + ), + } +} + +impl InstanceAndVmmUpdate { + pub fn new( + instance_id: Uuid, + new_instance_runtime_state: InstanceRuntimeState, + vmm_id: Uuid, + new_vmm_runtime_state: VmmRuntimeState, + ) -> Self { + let instance_find = Box::new( + instance_dsl::instance + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::id), + ); + + let vmm_find = Box::new( + vmm_dsl::vmm.filter(vmm_dsl::id.eq(vmm_id)).select(vmm_dsl::id), + ); + + let instance_update = Box::new( + diesel::update(instance_dsl::instance) + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .filter( + instance_dsl::state_generation + .lt(new_instance_runtime_state.gen), + ) + .set(new_instance_runtime_state), + ); + + let vmm_update = Box::new( + diesel::update(vmm_dsl::vmm) + .filter(vmm_dsl::time_deleted.is_null()) + .filter(vmm_dsl::id.eq(vmm_id)) + .filter(vmm_dsl::state_generation.lt(new_vmm_runtime_state.gen)) + .set(new_vmm_runtime_state), + ); + + Self { instance_find, vmm_find, instance_update, vmm_update } + } + + pub async fn execute_and_check( + self, + conn: &(impl async_bb8_diesel::AsyncConnection + Sync), + ) -> Result { + let (vmm_found, vmm_updated, instance_found, instance_updated) = + self.get_result_async::<(Option, + Option, + Option, + Option)>(conn).await?; + + let instance_status = + compute_update_status(instance_found, instance_updated); + let vmm_status = compute_update_status(vmm_found, vmm_updated); + + Ok(InstanceAndVmmUpdateResult { instance_status, vmm_status }) + } +} + +impl QueryId for InstanceAndVmmUpdate { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl Query for InstanceAndVmmUpdate { + type SqlType = ( + Nullable, + Nullable, + Nullable, + Nullable, + ); +} + +impl RunQueryDsl for InstanceAndVmmUpdate {} + +impl QueryFragment for InstanceAndVmmUpdate { + fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> QueryResult<()> { + out.push_sql("WITH instance_found AS (SELECT ("); + self.instance_find.walk_ast(out.reborrow())?; + out.push_sql(") AS id), "); + + out.push_sql("vmm_found AS (SELECT ("); + self.vmm_find.walk_ast(out.reborrow())?; + out.push_sql(") AS id), "); + + out.push_sql("instance_updated AS ("); + self.instance_update.walk_ast(out.reborrow())?; + out.push_sql(" RETURNING id), "); + + out.push_sql("vmm_updated AS ("); + self.vmm_update.walk_ast(out.reborrow())?; + out.push_sql(" RETURNING id), "); + + out.push_sql("vmm_result AS ("); + out.push_sql("SELECT vmm_found."); + out.push_identifier(vmm_dsl::id::NAME)?; + out.push_sql(" AS found, vmm_updated."); + out.push_identifier(vmm_dsl::id::NAME)?; + out.push_sql(" AS updated"); + out.push_sql(" FROM vmm_found LEFT JOIN vmm_updated ON vmm_found."); + out.push_identifier(vmm_dsl::id::NAME)?; + out.push_sql(" = vmm_updated."); + out.push_identifier(vmm_dsl::id::NAME)?; + out.push_sql("), "); + + out.push_sql("instance_result AS ("); + out.push_sql("SELECT instance_found."); + out.push_identifier(instance_dsl::id::NAME)?; + out.push_sql(" AS found, instance_updated."); + out.push_identifier(instance_dsl::id::NAME)?; + out.push_sql(" AS updated"); + out.push_sql( + " FROM instance_found LEFT JOIN instance_updated ON instance_found.", + ); + out.push_identifier(instance_dsl::id::NAME)?; + out.push_sql(" = instance_updated."); + out.push_identifier(instance_dsl::id::NAME)?; + out.push_sql(") "); + + out.push_sql("SELECT vmm_result.found, vmm_result.updated, "); + out.push_sql("instance_result.found, instance_result.updated "); + out.push_sql("FROM vmm_result, instance_result;"); + + Ok(()) + } +} diff --git a/nexus/db-queries/src/db/queries/mod.rs b/nexus/db-queries/src/db/queries/mod.rs index f91b54fb69..cd48be61e3 100644 --- a/nexus/db-queries/src/db/queries/mod.rs +++ b/nexus/db-queries/src/db/queries/mod.rs @@ -7,6 +7,7 @@ pub mod disk; pub mod external_ip; +pub mod instance; pub mod ip_pool; #[macro_use] mod next_item; diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs index bac2610b41..84a81a7b7a 100644 --- a/nexus/db-queries/src/db/queries/network_interface.rs +++ b/nexus/db-queries/src/db/queries/network_interface.rs @@ -60,6 +60,11 @@ lazy_static::lazy_static! { static ref INSTANCE_DESTROYED: db::model::InstanceState = db::model::InstanceState(external::InstanceState::Destroyed); + // A sentinel value for the instance state when the instance has an active + // VMM, irrespective of that VMM's actual state. + static ref INSTANCE_RUNNING: db::model::InstanceState = + db::model::InstanceState(external::InstanceState::Running); + static ref NO_INSTANCE_SENTINEL_STRING: String = String::from(NO_INSTANCE_SENTINEL); @@ -1273,7 +1278,10 @@ const INSTANCE_FROM_CLAUSE: InstanceFromClause = InstanceFromClause::new(); // -- Identify the state of the instance // ( // SELECT -// state +// CASE +// WHEN active_propolis_id IS NULL THEN state +// ELSE 'running' +// END // FROM // instance // WHERE @@ -1291,9 +1299,19 @@ const INSTANCE_FROM_CLAUSE: InstanceFromClause = InstanceFromClause::new(); // ``` // // This uses the familiar cast-fail trick to select the instance's UUID if the -// instance is in a state that can be altered, or a sentinel of `'running'` if -// not. It also ensures the instance exists at all with the sentinel -// `'no-instance'`. +// instance is in a state that allows network interfaces to be altered or +// produce a cast error if they cannot. The COALESCE statement and its innards +// yield the following state string: +// +// - 'destroyed' if the instance is not found at all +// - 'running' if the instance is found and has an active VMM (this forbids +// network interface changes irrespective of that VMM's actual state) +// - the instance's `state` otherwise +// +// If this produces 'stopped', 'creating', or (if applicable) 'failed', the +// outer CASE returns the instance ID as a string, which casts to a UUID. The +// 'destroyed' and 'bad-state' cases return non-UUID strings that cause a cast +// failure that can be caught and interpreted as a specific class of error. // // 'failed' is conditionally an accepted state: it would not be accepted as part // of InsertQuery, but it should be as part of DeleteQuery (for example if the @@ -1301,10 +1319,10 @@ const INSTANCE_FROM_CLAUSE: InstanceFromClause = InstanceFromClause::new(); // // Note that 'stopped', 'failed', and 'creating' are considered valid states. // 'stopped' is used for most situations, especially client-facing, but -// 'creating' is critical for the instance-creation saga. When we first -// provision the instance, its in the 'creating' state until a sled agent -// responds telling us that the instance has actually been launched. This -// additional case supports adding interfaces during that provisioning process. +// 'creating' is critical for the instance-creation saga. When an instance is +// first provisioned, it remains in the 'creating' state until provisioning is +// copmleted and it transitions to 'stopped'; it is permissible to add +// interfaces during that provisioning process. fn push_instance_state_verification_subquery<'a>( instance_id: &'a Uuid, @@ -1313,7 +1331,13 @@ fn push_instance_state_verification_subquery<'a>( failed_ok: bool, ) -> QueryResult<()> { out.push_sql("CAST(CASE COALESCE((SELECT "); + out.push_sql("CASE WHEN "); + out.push_identifier(db::schema::instance::dsl::active_propolis_id::NAME)?; + out.push_sql(" IS NULL THEN "); out.push_identifier(db::schema::instance::dsl::state::NAME)?; + out.push_sql(" ELSE "); + out.push_bind_param::(&INSTANCE_RUNNING)?; + out.push_sql(" END "); out.push_sql(" FROM "); INSTANCE_FROM_CLAUSE.walk_ast(out.reborrow())?; out.push_sql(" WHERE "); @@ -1662,7 +1686,6 @@ mod tests { use crate::db::model::Project; use crate::db::model::VpcSubnet; use async_bb8_diesel::AsyncRunQueryDsl; - use chrono::Utc; use dropshot::test_util::LogContext; use ipnetwork::Ipv4Network; use ipnetwork::Ipv6Network; @@ -1674,14 +1697,11 @@ mod tests { use omicron_common::api::external; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Error; - use omicron_common::api::external::Generation; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::InstanceCpuCount; - use omicron_common::api::external::InstanceState; use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::Ipv6Net; use omicron_common::api::external::MacAddr; - use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_test_utils::dev; use omicron_test_utils::dev::db::CockroachInstance; use std::convert::TryInto; @@ -1716,25 +1736,8 @@ mod tests { disks: vec![], start: true, }; - let runtime = InstanceRuntimeState { - run_state: InstanceState::Creating, - sled_id: Uuid::new_v4(), - propolis_id: Uuid::new_v4(), - dst_propolis_id: None, - propolis_addr: Some(std::net::SocketAddr::new( - "::1".parse().unwrap(), - 12400, - )), - migration_id: None, - propolis_gen: Generation::new(), - hostname: params.hostname.clone(), - memory: params.memory, - ncpus: params.ncpus, - gen: Generation::new(), - time_updated: Utc::now(), - }; - let instance = - Instance::new(instance_id, project_id, ¶ms, runtime.into()); + + let instance = Instance::new(instance_id, project_id, ¶ms); let (.., authz_project) = LookupPath::new(&opctx, &db_datastore) .project_id(project_id) @@ -1768,14 +1771,41 @@ mod tests { state: external::InstanceState, ) -> Instance { let new_runtime = model::InstanceRuntimeState { - state: model::InstanceState::new(state), + nexus_state: model::InstanceState::new(state), + gen: instance.runtime_state.gen.next().into(), + ..instance.runtime_state.clone() + }; + let res = db_datastore + .instance_update_runtime(&instance.id(), &new_runtime) + .await; + assert!(matches!(res, Ok(true)), "Failed to change instance state"); + instance.runtime_state = new_runtime; + instance + } + + /// Sets or clears the active Propolis ID in the supplied instance record. + /// This can be used to exercise the "does this instance have an active + /// VMM?" test that determines in part whether an instance's network + /// interfaces can change. + /// + /// Note that this routine does not construct a VMM record for the + /// corresponding ID, so any functions that expect such a record to exist + /// will fail in strange and exciting ways. + async fn instance_set_active_vmm( + db_datastore: &DataStore, + mut instance: Instance, + propolis_id: Option, + ) -> Instance { + let new_runtime = model::InstanceRuntimeState { + propolis_id, gen: instance.runtime_state.gen.next().into(), ..instance.runtime_state.clone() }; + let res = db_datastore .instance_update_runtime(&instance.id(), &new_runtime) .await; - assert!(matches!(res, Ok(true)), "Failed to stop instance"); + assert!(matches!(res, Ok(true)), "Failed to change instance VMM ref"); instance.runtime_state = new_runtime; instance } @@ -1900,10 +1930,7 @@ mod tests { self.logctx.cleanup_successful(); } - async fn create_instance( - &self, - state: external::InstanceState, - ) -> Instance { + async fn create_stopped_instance(&self) -> Instance { instance_set_state( &self.db_datastore, create_instance( @@ -1912,7 +1939,28 @@ mod tests { &self.db_datastore, ) .await, - state, + external::InstanceState::Stopped, + ) + .await + } + + async fn create_running_instance(&self) -> Instance { + let instance = instance_set_state( + &self.db_datastore, + create_instance( + &self.opctx, + self.project_id, + &self.db_datastore, + ) + .await, + external::InstanceState::Starting, + ) + .await; + + instance_set_active_vmm( + &self.db_datastore, + instance, + Some(Uuid::new_v4()), ) .await } @@ -1922,8 +1970,7 @@ mod tests { async fn test_insert_running_instance_fails() { let context = TestContext::new("test_insert_running_instance_fails", 2).await; - let instance = - context.create_instance(external::InstanceState::Running).await; + let instance = context.create_running_instance().await; let instance_id = instance.id(); let requested_ip = "172.30.0.5".parse().unwrap(); let interface = IncompleteNetworkInterface::new_instance( @@ -1952,8 +1999,7 @@ mod tests { #[tokio::test] async fn test_insert_request_exact_ip() { let context = TestContext::new("test_insert_request_exact_ip", 2).await; - let instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; let instance_id = instance.id(); let requested_ip = "172.30.0.5".parse().unwrap(); let interface = IncompleteNetworkInterface::new_instance( @@ -2024,8 +2070,7 @@ mod tests { .skip(NUM_INITIAL_RESERVED_IP_ADDRESSES); for (i, expected_address) in addresses.take(2).enumerate() { - let instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; let interface = IncompleteNetworkInterface::new_instance( Uuid::new_v4(), instance.id(), @@ -2063,10 +2108,8 @@ mod tests { let context = TestContext::new("test_insert_request_same_ip_fails", 2).await; - let instance = - context.create_instance(external::InstanceState::Stopped).await; - let new_instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; + let new_instance = context.create_stopped_instance().await; // Insert an interface on the first instance. let interface = IncompleteNetworkInterface::new_instance( @@ -2194,8 +2237,7 @@ mod tests { async fn test_insert_with_duplicate_name_fails() { let context = TestContext::new("test_insert_with_duplicate_name_fails", 2).await; - let instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; let interface = IncompleteNetworkInterface::new_instance( Uuid::new_v4(), instance.id(), @@ -2244,8 +2286,7 @@ mod tests { async fn test_insert_same_vpc_subnet_fails() { let context = TestContext::new("test_insert_same_vpc_subnet_fails", 2).await; - let instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; let interface = IncompleteNetworkInterface::new_instance( Uuid::new_v4(), instance.id(), @@ -2288,8 +2329,7 @@ mod tests { async fn test_insert_same_interface_fails() { let context = TestContext::new("test_insert_same_interface_fails", 2).await; - let instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; let interface = IncompleteNetworkInterface::new_instance( Uuid::new_v4(), instance.id(), @@ -2330,8 +2370,7 @@ mod tests { async fn test_insert_multiple_vpcs_fails() { let context = TestContext::new("test_insert_multiple_vpcs_fails", 2).await; - let instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; let interface = IncompleteNetworkInterface::new_instance( Uuid::new_v4(), instance.id(), @@ -2384,8 +2423,7 @@ mod tests { let context = TestContext::new("test_detect_ip_exhaustion", 2).await; let n_interfaces = context.net1.available_ipv4_addresses()[0]; for _ in 0..n_interfaces { - let instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; let interface = IncompleteNetworkInterface::new_instance( Uuid::new_v4(), instance.id(), @@ -2443,8 +2481,7 @@ mod tests { let context = TestContext::new("test_insert_multiple_vpc_subnets_succeeds", 2) .await; - let instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; for (i, subnet) in context.net1.subnets.iter().enumerate() { let interface = IncompleteNetworkInterface::new_instance( Uuid::new_v4(), @@ -2509,8 +2546,7 @@ mod tests { MAX_NICS as u8 + 1, ) .await; - let instance = - context.create_instance(external::InstanceState::Stopped).await; + let instance = context.create_stopped_instance().await; for slot in 0..MAX_NICS { let subnet = &context.net1.subnets[slot]; let interface = IncompleteNetworkInterface::new_instance( diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index f07ceae4a0..592e1f0492 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -19,9 +19,9 @@ use futures::{FutureExt, SinkExt, StreamExt}; use nexus_db_model::IpKind; use nexus_db_queries::authn; use nexus_db_queries::authz; -use nexus_db_queries::authz::ApiResource; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; +use nexus_db_queries::db::datastore::InstanceAndActiveVmm; use nexus_db_queries::db::identity::Resource; use nexus_db_queries::db::lookup; use nexus_db_queries::db::lookup::LookupPath; @@ -47,11 +47,11 @@ use propolis_client::support::InstanceSerialConsoleHelper; use propolis_client::support::WSClientOffset; use propolis_client::support::WebSocketStream; use sled_agent_client::types::InstanceMigrationSourceParams; +use sled_agent_client::types::InstanceMigrationTargetParams; +use sled_agent_client::types::InstanceProperties; use sled_agent_client::types::InstancePutMigrationIdsBody; use sled_agent_client::types::InstancePutStateBody; -use sled_agent_client::types::InstanceStateRequested; use sled_agent_client::types::SourceNatConfig; -use sled_agent_client::Client as SledAgentClient; use std::net::SocketAddr; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; @@ -59,9 +59,39 @@ use uuid::Uuid; const MAX_KEYS_PER_INSTANCE: u32 = 8; -pub(crate) enum WriteBackUpdatedInstance { - WriteBack, - Drop, +/// The kinds of state changes that can be requested of an instance's current +/// VMM (i.e. the VMM pointed to be the instance's `propolis_id` field). +pub(crate) enum InstanceStateChangeRequest { + Run, + Reboot, + Stop, + Migrate(InstanceMigrationTargetParams), +} + +impl From + for sled_agent_client::types::InstanceStateRequested +{ + fn from(value: InstanceStateChangeRequest) -> Self { + match value { + InstanceStateChangeRequest::Run => Self::Running, + InstanceStateChangeRequest::Reboot => Self::Reboot, + InstanceStateChangeRequest::Stop => Self::Stopped, + InstanceStateChangeRequest::Migrate(params) => { + Self::MigrationTarget(params) + } + } + } +} + +/// The actions that can be taken in response to an +/// [`InstanceStateChangeRequest`]. +enum InstanceStateChangeRequestAction { + /// The instance is already in the correct state, so no action is needed. + AlreadyDone, + + /// Request the appropriate state change from the sled with the specified + /// UUID. + SendToSled(Uuid), } impl super::Nexus { @@ -109,7 +139,7 @@ impl super::Nexus { opctx: &OpContext, project_lookup: &lookup::Project<'_>, params: ¶ms::InstanceCreate, - ) -> CreateResult { + ) -> CreateResult { let (.., authz_project) = project_lookup.lookup_for(authz::Action::CreateChild).await?; @@ -190,7 +220,7 @@ impl super::Nexus { }); } - // Reject instances where the memory is greated than the limit + // Reject instances where the memory is greater than the limit if params.memory.to_bytes() > MAX_MEMORY_BYTES_PER_INSTANCE { return Err(Error::InvalidValue { label: String::from("size"), @@ -221,46 +251,38 @@ impl super::Nexus { .map_err(|e| Error::internal_error(&format!("{:#}", &e))) .internal_context("looking up output from instance create saga")?; - // TODO-correctness TODO-robustness TODO-design It's not quite correct - // to take this instance id and look it up again. It's possible that - // it's been modified or even deleted since the saga executed. In that - // case, we might return a different state of the Instance than the one - // that the user created or even fail with a 404! Both of those are - // wrong behavior -- we should be returning the very instance that the - // user created. - // - // How can we fix this? Right now we have internal representations like - // Instance and analaogous end-user-facing representations like - // Instance. The former is not even serializable. The saga - // _could_ emit the View version, but that's not great for two (related) - // reasons: (1) other sagas might want to provision instances and get - // back the internal representation to do other things with the - // newly-created instance, and (2) even within a saga, it would be - // useful to pass a single Instance representation along the saga, - // but they probably would want the internal representation, not the - // view. - // - // The saga could emit an Instance directly. Today, Instance - // etc. aren't supposed to even be serializable -- we wanted to be able - // to have other datastore state there if needed. We could have a third - // InstanceInternalView...but that's starting to feel pedantic. We - // could just make Instance serializable, store that, and call it a - // day. Does it matter that we might have many copies of the same - // objects in memory? - // - // If we make these serializable, it would be nice if we could leverage - // the type system to ensure that we never accidentally send them out a - // dropshot endpoint. (On the other hand, maybe we _do_ want to do - // that, for internal interfaces! Can we do this on a - // per-dropshot-server-basis?) + // If the caller asked to start the instance, kick off that saga. + // There's a window in which the instance is stopped and can be deleted, + // so this is not guaranteed to succeed, and its result should not + // affect the result of the attempt to create the instance. + if params.start { + let lookup = LookupPath::new(opctx, &self.db_datastore) + .instance_id(instance_id); + + let start_result = self.instance_start(opctx, &lookup).await; + if let Err(e) = start_result { + info!(self.log, "failed to start newly-created instance"; + "instance_id" => %instance_id, + "error" => ?e); + } + } + + // TODO: This operation should return the instance as it was created. + // Refetching the instance state here won't return that version of the + // instance if its state changed between the time the saga finished and + // the time this lookup was performed. // - // TODO Even worse, post-authz, we do two lookups here instead of one. - // Maybe sagas should be able to emit `authz::Instance`-type objects. - let (.., db_instance) = LookupPath::new(opctx, &self.db_datastore) + // Because the create saga has to synthesize an instance record (and + // possibly a VMM record), and these are serializable, it should be + // possible to yank the outputs out of the appropriate saga steps and + // return them here. + + let (.., authz_instance) = LookupPath::new(opctx, &self.db_datastore) .instance_id(instance_id) - .fetch() + .lookup_for(authz::Action::Read) .await?; - Ok(db_instance) + + self.db_datastore.instance_fetch_with_vmm(opctx, &authz_instance).await } pub(crate) async fn instance_list( @@ -268,7 +290,7 @@ impl super::Nexus { opctx: &OpContext, project_lookup: &lookup::Project<'_>, pagparams: &PaginatedBy<'_>, - ) -> ListResultVec { + ) -> ListResultVec { let (.., authz_project) = project_lookup.lookup_for(authz::Action::ListChildren).await?; self.db_datastore.instance_list(opctx, &authz_project, pagparams).await @@ -314,30 +336,40 @@ impl super::Nexus { opctx: &OpContext, instance_lookup: &lookup::Instance<'_>, params: params::InstanceMigrate, - ) -> UpdateResult { - let (.., authz_instance, db_instance) = - instance_lookup.fetch_for(authz::Action::Modify).await?; + ) -> UpdateResult { + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Modify).await?; + + let state = self + .db_datastore + .instance_fetch_with_vmm(opctx, &authz_instance) + .await?; + let (instance, vmm) = (state.instance(), state.vmm()); - if db_instance.runtime().state.0 != InstanceState::Running { + if vmm.is_none() + || vmm.as_ref().unwrap().runtime.state.0 != InstanceState::Running + { return Err(Error::invalid_request( "instance must be running before it can migrate", )); } - if db_instance.runtime().sled_id == params.dst_sled_id { + let vmm = vmm.as_ref().unwrap(); + if vmm.sled_id == params.dst_sled_id { return Err(Error::invalid_request( "instance is already running on destination sled", )); } - if db_instance.runtime().migration_id.is_some() { + if instance.runtime().migration_id.is_some() { return Err(Error::unavail("instance is already migrating")); } // Kick off the migration saga let saga_params = sagas::instance_migrate::Params { serialized_authn: authn::saga::Serialized::for_opctx(opctx), - instance: db_instance, + instance: instance.clone(), + src_vmm: vmm.clone(), migrate_params: params, }; self.execute_saga::( @@ -348,7 +380,7 @@ impl super::Nexus { // TODO correctness TODO robustness TODO design // Should we lookup the instance again here? // See comment in project_create_instance. - self.db_datastore.instance_refetch(opctx, &authz_instance).await + self.db_datastore.instance_fetch_with_vmm(opctx, &authz_instance).await } /// Attempts to set the migration IDs for the supplied instance via the @@ -370,23 +402,24 @@ impl super::Nexus { &self, opctx: &OpContext, instance_id: Uuid, - db_instance: &db::model::Instance, + sled_id: Uuid, + prev_instance_runtime: &db::model::InstanceRuntimeState, migration_params: InstanceMigrationSourceParams, ) -> UpdateResult { - assert!(db_instance.runtime().migration_id.is_none()); - assert!(db_instance.runtime().dst_propolis_id.is_none()); + assert!(prev_instance_runtime.migration_id.is_none()); + assert!(prev_instance_runtime.dst_propolis_id.is_none()); let (.., authz_instance) = LookupPath::new(opctx, &self.db_datastore) .instance_id(instance_id) .lookup_for(authz::Action::Modify) .await?; - let sa = self.instance_sled(&db_instance).await?; + let sa = self.sled_client(&sled_id).await?; let instance_put_result = sa .instance_put_migration_ids( &instance_id, &InstancePutMigrationIdsBody { - old_runtime: db_instance.runtime().clone().into(), + old_runtime: prev_instance_runtime.clone().into(), migration_params: Some(migration_params), }, ) @@ -397,8 +430,12 @@ impl super::Nexus { // outright fails, this operation fails. If the operation nominally // succeeds but nothing was updated, this action is outdated and the // caller should not proceed with migration. - let updated = self - .handle_instance_put_result(&db_instance, instance_put_result) + let (updated, _) = self + .handle_instance_put_result( + &instance_id, + prev_instance_runtime, + instance_put_result.map(|state| state.map(Into::into)), + ) .await?; if updated { @@ -431,25 +468,30 @@ impl super::Nexus { pub(crate) async fn instance_clear_migration_ids( &self, instance_id: Uuid, - db_instance: &db::model::Instance, + sled_id: Uuid, + prev_instance_runtime: &db::model::InstanceRuntimeState, ) -> Result<(), Error> { - assert!(db_instance.runtime().migration_id.is_some()); - assert!(db_instance.runtime().dst_propolis_id.is_some()); + assert!(prev_instance_runtime.migration_id.is_some()); + assert!(prev_instance_runtime.dst_propolis_id.is_some()); - let sa = self.instance_sled(&db_instance).await?; + let sa = self.sled_client(&sled_id).await?; let instance_put_result = sa .instance_put_migration_ids( &instance_id, &InstancePutMigrationIdsBody { - old_runtime: db_instance.runtime().clone().into(), + old_runtime: prev_instance_runtime.clone().into(), migration_params: None, }, ) .await .map(|res| Some(res.into_inner())); - self.handle_instance_put_result(&db_instance, instance_put_result) - .await?; + self.handle_instance_put_result( + &instance_id, + prev_instance_runtime, + instance_put_result.map(|state| state.map(Into::into)), + ) + .await?; Ok(()) } @@ -459,16 +501,24 @@ impl super::Nexus { &self, opctx: &OpContext, instance_lookup: &lookup::Instance<'_>, - ) -> UpdateResult { - let (.., authz_instance, db_instance) = instance_lookup.fetch().await?; + ) -> UpdateResult { + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Modify).await?; + + let state = self + .db_datastore + .instance_fetch_with_vmm(opctx, &authz_instance) + .await?; + self.instance_request_state( opctx, &authz_instance, - &db_instance, - InstanceStateRequested::Reboot, + state.instance(), + state.vmm(), + InstanceStateChangeRequest::Reboot, ) .await?; - self.db_datastore.instance_refetch(opctx, &authz_instance).await + self.db_datastore.instance_fetch_with_vmm(opctx, &authz_instance).await } /// Attempts to start an instance if it is currently stopped. @@ -476,42 +526,53 @@ impl super::Nexus { self: &Arc, opctx: &OpContext, instance_lookup: &lookup::Instance<'_>, - ) -> UpdateResult { - let (.., authz_instance, db_instance) = - instance_lookup.fetch_for(authz::Action::Modify).await?; + ) -> UpdateResult { + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Modify).await?; - // If the instance is already starting or running, succeed immediately - // for idempotency. If the instance is stopped, try to start it. In all - // other cases return an error describing the state conflict. - // - // The "Creating" state is not permitted here (even though a request to - // create can include a request to start the instance) because an - // instance that is still being created may not be ready to start yet - // (e.g. its disks may not yet be attached). - // - // If the instance is stopped, the start saga will try to change the - // instance's state to Starting and increment the instance's state - // generation number. If this increment fails (because someone else has - // changed the state), the saga fails. See the saga comments for more - // details on how this synchronization works. - match db_instance.runtime_state.state.0 { - InstanceState::Starting | InstanceState::Running => { - return Ok(db_instance) - } - InstanceState::Stopped => {} - _ => { - return Err(Error::conflict(&format!( - "instance is in state {} but must be {} to be started", - db_instance.runtime_state.state.0, - InstanceState::Stopped - ))) + let state = self + .db_datastore + .instance_fetch_with_vmm(opctx, &authz_instance) + .await?; + let (instance, vmm) = (state.instance(), state.vmm()); + + if let Some(vmm) = vmm { + match vmm.runtime.state.0 { + InstanceState::Starting + | InstanceState::Running + | InstanceState::Rebooting => { + debug!(self.log, "asked to start an active instance"; + "instance_id" => %authz_instance.id()); + + return Ok(state); + } + InstanceState::Stopped => { + let propolis_id = instance + .runtime() + .propolis_id + .expect("needed a VMM ID to fetch a VMM record"); + error!(self.log, + "instance is stopped but still has an active VMM"; + "instance_id" => %authz_instance.id(), + "propolis_id" => %propolis_id); + + return Err(Error::internal_error( + "instance is stopped but still has an active VMM", + )); + } + _ => { + return Err(Error::conflict(&format!( + "instance is in state {} but must be {} to be started", + vmm.runtime.state.0, + InstanceState::Stopped + ))); + } } } let saga_params = sagas::instance_start::Params { serialized_authn: authn::saga::Serialized::for_opctx(opctx), - instance: db_instance, - ensure_network: true, + db_instance: instance.clone(), }; self.execute_saga::( @@ -519,7 +580,7 @@ impl super::Nexus { ) .await?; - self.db_datastore.instance_refetch(opctx, &authz_instance).await + self.db_datastore.instance_fetch_with_vmm(opctx, &authz_instance).await } /// Make sure the given Instance is stopped. @@ -527,16 +588,25 @@ impl super::Nexus { &self, opctx: &OpContext, instance_lookup: &lookup::Instance<'_>, - ) -> UpdateResult { - let (.., authz_instance, db_instance) = instance_lookup.fetch().await?; + ) -> UpdateResult { + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Modify).await?; + + let state = self + .db_datastore + .instance_fetch_with_vmm(opctx, &authz_instance) + .await?; + self.instance_request_state( opctx, &authz_instance, - &db_instance, - InstanceStateRequested::Stopped, + state.instance(), + state.vmm(), + InstanceStateChangeRequest::Stop, ) .await?; - self.db_datastore.instance_refetch(opctx, &authz_instance).await + + self.db_datastore.instance_fetch_with_vmm(opctx, &authz_instance).await } /// Idempotently ensures that the sled specified in `db_instance` does not @@ -546,76 +616,165 @@ impl super::Nexus { &self, opctx: &OpContext, authz_instance: &authz::Instance, - db_instance: &db::model::Instance, - write_back: WriteBackUpdatedInstance, + sled_id: &Uuid, + prev_instance_runtime: &db::model::InstanceRuntimeState, ) -> Result<(), Error> { opctx.authorize(authz::Action::Modify, authz_instance).await?; - let sa = self.instance_sled(&db_instance).await?; + let sa = self.sled_client(&sled_id).await?; let result = sa - .instance_unregister(&db_instance.id()) + .instance_unregister(&authz_instance.id()) .await .map(|res| res.into_inner().updated_runtime); - match write_back { - WriteBackUpdatedInstance::WriteBack => self - .handle_instance_put_result(db_instance, result) - .await - .map(|_| ()), - WriteBackUpdatedInstance::Drop => { - result?; - Ok(()) - } - } + self.handle_instance_put_result( + &authz_instance.id(), + prev_instance_runtime, + result.map(|state| state.map(Into::into)), + ) + .await + .map(|_| ()) } - /// Returns the SledAgentClient for the host where this Instance is running. - pub(crate) async fn instance_sled( + /// Determines the action to take on an instance's active VMM given a + /// request to change its state. + /// + /// # Arguments + /// + /// - instance_state: The prior state of the instance as recorded in CRDB + /// and obtained by the caller. + /// - vmm_state: The prior state of the instance's active VMM as recorded in + /// CRDB and obtained by the caller. `None` if the instance has no active + /// VMM. + /// - requested: The state change being requested. + /// + /// # Return value + /// + /// - `Ok(action)` if the request is allowed to proceed. The result payload + /// specifies how to handle the request. + /// - `Err` if the request should be denied. + fn select_runtime_change_action( &self, - instance: &db::model::Instance, - ) -> Result, Error> { - let sa_id = &instance.runtime().sled_id; - self.sled_client(&sa_id).await - } + instance_state: &db::model::Instance, + vmm_state: &Option, + requested: &InstanceStateChangeRequest, + ) -> Result { + let effective_state = if let Some(vmm) = vmm_state { + vmm.runtime.state.0 + } else { + instance_state.runtime().nexus_state.0 + }; - fn check_runtime_change_allowed( - &self, - runtime: &nexus::InstanceRuntimeState, - requested: &InstanceStateRequested, - ) -> Result<(), Error> { - // Users are allowed to request a start or stop even if the instance is - // already in the desired state (or moving to it), and we will issue a - // request to the SA to make the state change in these cases in case the - // runtime state we saw here was stale. - // - // Users cannot change the state of a failed or destroyed instance. - // TODO(#2825): Failed instances should be allowed to stop. - // - // Migrating instances can't change state until they're done migrating, - // but for idempotency, a request to make an incarnation of an instance - // into a migration target is allowed if the incarnation is already a - // migration target. - let allowed = match runtime.run_state { - InstanceState::Creating => true, - InstanceState::Starting => true, - InstanceState::Running => true, - InstanceState::Stopping => true, - InstanceState::Stopped => true, - InstanceState::Rebooting => true, - InstanceState::Migrating => { - matches!(requested, InstanceStateRequested::MigrationTarget(_)) + // Requests that operate on active instances have to be directed to the + // instance's current sled agent. If there is none, the request needs to + // be handled specially based on its type. + let sled_id = if let Some(vmm) = vmm_state { + vmm.sled_id + } else { + match effective_state { + // If there's no active sled because the instance is stopped, + // allow requests to stop to succeed silently for idempotency, + // but reject requests to do anything else. + InstanceState::Stopped => match requested { + InstanceStateChangeRequest::Run => { + return Err(Error::invalid_request(&format!( + "cannot run an instance in state {} with no VMM", + effective_state + ))) + } + InstanceStateChangeRequest::Stop => { + return Ok(InstanceStateChangeRequestAction::AlreadyDone); + } + InstanceStateChangeRequest::Reboot => { + return Err(Error::invalid_request(&format!( + "cannot reboot an instance in state {} with no VMM", + effective_state + ))) + } + InstanceStateChangeRequest::Migrate(_) => { + return Err(Error::invalid_request(&format!( + "cannot migrate an instance in state {} with no VMM", + effective_state + ))) + } + }, + + // If the instance is still being created (such that it hasn't + // even begun to start yet), no runtime state change is valid. + // Return a specific error message explaining the problem. + InstanceState::Creating => { + return Err(Error::invalid_request( + "cannot change instance state while it is \ + still being created" + )) + } + + // If the instance has no sled beacuse it's been destroyed or + // has fallen over, reject the state change. + // + // TODO(#2825): Failed instances should be allowed to stop, but + // this requires a special action because there is no sled to + // send the request to. + InstanceState::Failed | InstanceState::Destroyed => { + return Err(Error::invalid_request(&format!( + "instance state cannot be changed from {}", + effective_state + ))) + } + + // In other states, the instance should have a sled, and an + // internal invariant has been violated if it doesn't have one. + _ => { + error!(self.log, "instance has no sled but isn't halted"; + "instance_id" => %instance_state.id(), + "state" => ?effective_state); + + return Err(Error::internal_error( + "instance is active but not resident on a sled" + )); + } } - InstanceState::Repairing => false, - InstanceState::Failed => false, - InstanceState::Destroyed => false, + }; + + // The instance has an active sled. Allow the sled agent to decide how + // to handle the request unless the instance is being recovered or the + // underlying VMM has been destroyed. + // + // TODO(#2825): Failed instances should be allowed to stop. See above. + let allowed = match requested { + InstanceStateChangeRequest::Run + | InstanceStateChangeRequest::Reboot + | InstanceStateChangeRequest::Stop => match effective_state { + InstanceState::Creating + | InstanceState::Starting + | InstanceState::Running + | InstanceState::Stopping + | InstanceState::Stopped + | InstanceState::Rebooting + | InstanceState::Migrating => true, + InstanceState::Repairing | InstanceState::Failed => false, + InstanceState::Destroyed => false, + }, + InstanceStateChangeRequest::Migrate(_) => match effective_state { + InstanceState::Running + | InstanceState::Rebooting + | InstanceState::Migrating => true, + InstanceState::Creating + | InstanceState::Starting + | InstanceState::Stopping + | InstanceState::Stopped + | InstanceState::Repairing + | InstanceState::Failed + | InstanceState::Destroyed => false, + }, }; if allowed { - Ok(()) + Ok(InstanceStateChangeRequestAction::SendToSled(sled_id)) } else { Err(Error::InvalidRequest { message: format!( "instance state cannot be changed from state \"{}\"", - runtime.run_state + effective_state ), }) } @@ -625,27 +784,39 @@ impl super::Nexus { &self, opctx: &OpContext, authz_instance: &authz::Instance, - db_instance: &db::model::Instance, - requested: InstanceStateRequested, + prev_instance_state: &db::model::Instance, + prev_vmm_state: &Option, + requested: InstanceStateChangeRequest, ) -> Result<(), Error> { opctx.authorize(authz::Action::Modify, authz_instance).await?; - self.check_runtime_change_allowed( - &db_instance.runtime().clone().into(), - &requested, - )?; + let instance_id = authz_instance.id(); - let sa = self.instance_sled(&db_instance).await?; - let instance_put_result = sa - .instance_put_state( - &db_instance.id(), - &InstancePutStateBody { state: requested }, - ) - .await - .map(|res| res.into_inner().updated_runtime); + match self.select_runtime_change_action( + prev_instance_state, + prev_vmm_state, + &requested, + )? { + InstanceStateChangeRequestAction::AlreadyDone => Ok(()), + InstanceStateChangeRequestAction::SendToSled(sled_id) => { + let sa = self.sled_client(&sled_id).await?; + let instance_put_result = sa + .instance_put_state( + &instance_id, + &InstancePutStateBody { state: requested.into() }, + ) + .await + .map(|res| res.into_inner().updated_runtime) + .map(|state| state.map(Into::into)); - self.handle_instance_put_result(db_instance, instance_put_result) - .await - .map(|_| ()) + self.handle_instance_put_result( + &instance_id, + prev_instance_state.runtime(), + instance_put_result, + ) + .await + .map(|_| ()) + } + } } /// Modifies the runtime state of the Instance as requested. This generally @@ -655,6 +826,8 @@ impl super::Nexus { opctx: &OpContext, authz_instance: &authz::Instance, db_instance: &db::model::Instance, + propolis_id: &Uuid, + initial_vmm: &db::model::Vmm, ) -> Result<(), Error> { opctx.authorize(authz::Action::Modify, authz_instance).await?; @@ -684,7 +857,7 @@ impl super::Nexus { error!(self.log, "attached disk has no PCI slot assignment"; "disk_id" => %disk.id(), "disk_name" => disk.name().to_string(), - "instance" => ?disk.runtime_state.attach_instance_id); + "instance_id" => ?disk.runtime_state.attach_instance_id); return Err(Error::internal_error(&format!( "disk {} is attached but has no PCI slot assignment", @@ -805,9 +978,11 @@ impl super::Nexus { // beat us to it. let instance_hardware = sled_agent_client::types::InstanceHardware { - runtime: sled_agent_client::types::InstanceRuntimeState::from( - db_instance.runtime().clone(), - ), + properties: InstanceProperties { + ncpus: db_instance.ncpus.into(), + memory: db_instance.memory.into(), + hostname: db_instance.hostname.clone(), + }, nics, source_nat, external_ips, @@ -819,21 +994,32 @@ impl super::Nexus { )), }; - let sa = self.instance_sled(&db_instance).await?; - + let sa = self.sled_client(&initial_vmm.sled_id).await?; let instance_register_result = sa .instance_register( &db_instance.id(), &sled_agent_client::types::InstanceEnsureBody { - initial: instance_hardware, + hardware: instance_hardware, + instance_runtime: db_instance.runtime().clone().into(), + vmm_runtime: initial_vmm.clone().into(), + propolis_id: *propolis_id, + propolis_addr: SocketAddr::new( + initial_vmm.propolis_ip.ip(), + PROPOLIS_PORT, + ) + .to_string(), }, ) .await .map(|res| Some(res.into_inner())); - self.handle_instance_put_result(db_instance, instance_register_result) - .await - .map(|_| ()) + self.handle_instance_put_result( + &db_instance.id(), + db_instance.runtime(), + instance_register_result.map(|state| state.map(Into::into)), + ) + .await + .map(|_| ()) } /// Updates an instance's CRDB record based on the result of a call to sled @@ -860,34 +1046,38 @@ impl super::Nexus { /// error while trying to update CRDB. async fn handle_instance_put_result( &self, - db_instance: &db::model::Instance, + instance_id: &Uuid, + prev_instance_runtime: &db::model::InstanceRuntimeState, result: Result< - Option, + Option, sled_agent_client::Error, >, - ) -> Result { + ) -> Result<(bool, bool), Error> { slog::debug!(&self.log, "Handling sled agent instance PUT result"; + "instance_id" => %instance_id, "result" => ?result); match result { - Ok(Some(new_runtime)) => { - let new_runtime: nexus::InstanceRuntimeState = - new_runtime.into(); - + Ok(Some(new_state)) => { let update_result = self .db_datastore - .instance_update_runtime( - &db_instance.id(), - &new_runtime.into(), + .instance_and_vmm_update_runtime( + instance_id, + &new_state.instance_state.into(), + &new_state.propolis_id, + &new_state.vmm_state.into(), ) .await; slog::debug!(&self.log, "Attempted DB update after instance PUT"; + "instance_id" => %instance_id, + "propolis_id" => %new_state.propolis_id, "result" => ?update_result); + update_result } - Ok(None) => Ok(false), + Ok(None) => Ok((false, false)), Err(e) => { // The sled-agent has told us that it can't do what we // requested, but does that mean a failure? One example would be @@ -898,13 +1088,15 @@ impl super::Nexus { // // Without a richer error type, let the sled-agent tell Nexus // what to do with status codes. - error!(self.log, "saw {} from instance_put!", e); + error!(self.log, "received error from instance PUT"; + "instance_id" => %instance_id, + "error" => ?e); // Convert to the Omicron API error type. // - // N.B. The match below assumes that this conversion will turn - // any 400-level error status from sled agent into an - // `Error::InvalidRequest`. + // TODO(#3238): This is an extremely lossy conversion: if the + // operation failed without getting a response from sled agent, + // this unconditionally converts to Error::InternalError. let e = e.into(); match &e { @@ -914,28 +1106,41 @@ impl super::Nexus { // Internal server error (or anything else) should change // the instance state to failed, we don't know what state // the instance is in. + // + // TODO(#4226): This logic needs to be revisited: + // - Some errors that don't get classified as + // Error::InvalidRequest (timeouts, disconnections due to + // network weather, etc.) are not necessarily fatal to the + // instance and shouldn't mark it as Failed. + // - If the instance still has a running VMM, this operation + // won't terminate it or reclaim its resources. (The + // resources will be reclaimed if the sled later reports + // that the VMM is gone, however.) _ => { let new_runtime = db::model::InstanceRuntimeState { - state: db::model::InstanceState::new( + nexus_state: db::model::InstanceState::new( InstanceState::Failed, ), - gen: db_instance.runtime_state.gen.next().into(), - ..db_instance.runtime_state.clone() + + // TODO(#4226): Clearing the Propolis ID is required + // to allow the instance to be deleted, but this + // doesn't actually terminate the VMM (see above). + propolis_id: None, + gen: prev_instance_runtime.gen.next().into(), + ..prev_instance_runtime.clone() }; // XXX what if this fails? let result = self .db_datastore - .instance_update_runtime( - &db_instance.id(), - &new_runtime, - ) + .instance_update_runtime(&instance_id, &new_runtime) .await; error!( self.log, - "saw {:?} from setting InstanceState::Failed after bad instance_put", - result, + "attempted to set instance to Failed after bad put"; + "instance_id" => %instance_id, + "result" => ?result, ); Err(e) @@ -983,10 +1188,11 @@ impl super::Nexus { .await?; // TODO-v1: Write test to verify this case - // Because both instance and disk can be provided by ID it's possible for someone - // to specify resources from different projects. The lookups would resolve the resources - // (assuming the user had sufficient permissions on both) without verifying the shared hierarchy. - // To mitigate that we verify that their parent projects have the same ID. + // Because both instance and disk can be provided by ID it's possible + // for someone to specify resources from different projects. The lookups + // would resolve the resources (assuming the user had sufficient + // permissions on both) without verifying the shared hierarchy. To + // mitigate that we verify that their parent projects have the same ID. if authz_project.id() != authz_project_disk.id() { return Err(Error::InvalidRequest { message: "disk must be in the same project as the instance" @@ -1066,91 +1272,111 @@ impl super::Nexus { pub(crate) async fn notify_instance_updated( &self, opctx: &OpContext, - id: &Uuid, - new_runtime_state: &nexus::InstanceRuntimeState, + instance_id: &Uuid, + new_runtime_state: &nexus::SledInstanceState, ) -> Result<(), Error> { let log = &self.log; + let propolis_id = new_runtime_state.propolis_id; - slog::debug!(log, "received new runtime state from sled agent"; - "instance_id" => %id, - "runtime_state" => ?new_runtime_state); + info!(log, "received new runtime state from sled agent"; + "instance_id" => %instance_id, + "instance_state" => ?new_runtime_state.instance_state, + "propolis_id" => %propolis_id, + "vmm_state" => ?new_runtime_state.vmm_state); - // If the new state has a newer Propolis ID generation than the current - // instance state in CRDB, notify interested parties of this change. + // Update OPTE and Dendrite if the instance's active sled assignment + // changed or a migration was retired. If these actions fail, sled agent + // is expected to retry this update. // - // The synchronization rules here are as follows: + // This configuration must be updated before updating any state in CRDB + // so that, if the instance was migrating or has shut down, it will not + // appear to be able to migrate or start again until the appropriate + // networking state has been written. Without this interlock, another + // thread or another Nexus can race with this routine to write + // conflicting configuration. // - // - Sled agents own an instance's runtime state while an instance is - // running on a sled. Each sled agent prevents concurrent conflicting - // Propolis identifier updates from being sent until previous updates - // are processed. - // - Operations that can dispatch an instance to a brand-new sled (e.g. - // live migration) can only start if the appropriate instance runtime - // state fields are cleared in CRDB. For example, while a live - // migration is in progress, the instance's `migration_id` field will - // be non-NULL, and a new migration cannot start until it is cleared. - // This routine must notify recipients before writing new records - // back to CRDB so that these "locks" remain held until all - // notifications have been sent. Otherwise, Nexus might allow new - // operations to proceed that will produce system updates that might - // race with this one. - // - This work is not done in a saga. The presumption is instead that - // if any of these operations fail, the entire update will fail, and - // sled agent will retry the update. Unwinding on failure isn't needed - // because (a) any partially-applied configuration is correct - // configuration, (b) if the instance is migrating, it can't migrate - // again until this routine successfully updates configuration and - // writes an update back to CRDB, and (c) sled agent won't process any - // new instance state changes (e.g. a change that stops an instance) - // until this state change is successfully committed. - let (.., db_instance) = LookupPath::new(&opctx, &self.db_datastore) - .instance_id(*id) - .fetch_for(authz::Action::Read) - .await?; + // In the future, this should be replaced by a call to trigger a + // networking state update RPW. + let (.., authz_instance, db_instance) = + LookupPath::new(&opctx, &self.db_datastore) + .instance_id(*instance_id) + .fetch() + .await?; - if new_runtime_state.propolis_gen > *db_instance.runtime().propolis_gen - { - self.handle_instance_propolis_gen_change( - opctx, - new_runtime_state, - &db_instance, - ) - .await?; - } + self.ensure_updated_instance_network_config( + opctx, + &authz_instance, + db_instance.runtime(), + &new_runtime_state.instance_state, + ) + .await?; + // Write the new instance and VMM states back to CRDB. This needs to be + // done before trying to clean up the VMM, since the datastore will only + // allow a VMM to be marked as deleted if it is already in a terminal + // state. let result = self .db_datastore - .instance_update_runtime(id, &(new_runtime_state.clone().into())) + .instance_and_vmm_update_runtime( + instance_id, + &db::model::InstanceRuntimeState::from( + new_runtime_state.instance_state.clone(), + ), + &propolis_id, + &db::model::VmmRuntimeState::from( + new_runtime_state.vmm_state.clone(), + ), + ) .await; - match result { - Ok(true) => { - info!(log, "instance updated by sled agent"; - "instance_id" => %id, - "propolis_id" => %new_runtime_state.propolis_id, - "new_state" => %new_runtime_state.run_state); - Ok(()) + // If the VMM is now in a terminal state, make sure its resources get + // cleaned up. + if let Ok((_, true)) = result { + let propolis_terminated = matches!( + new_runtime_state.vmm_state.state, + InstanceState::Destroyed | InstanceState::Failed + ); + + if propolis_terminated { + info!(log, "vmm is terminated, cleaning up resources"; + "instance_id" => %instance_id, + "propolis_id" => %propolis_id); + + self.db_datastore + .sled_reservation_delete(opctx, propolis_id) + .await?; + + if !self + .db_datastore + .vmm_mark_deleted(opctx, &propolis_id) + .await? + { + warn!(log, "failed to mark vmm record as deleted"; + "instance_id" => %instance_id, + "propolis_id" => %propolis_id, + "vmm_state" => ?new_runtime_state.vmm_state); + } } + } - Ok(false) => { - info!(log, "instance update from sled agent ignored (old)"; - "instance_id" => %id, - "propolis_id" => %new_runtime_state.propolis_id, - "requested_state" => %new_runtime_state.run_state); + match result { + Ok((instance_updated, vmm_updated)) => { + info!(log, "instance and vmm updated by sled agent"; + "instance_id" => %instance_id, + "propolis_id" => %propolis_id, + "instance_updated" => instance_updated, + "vmm_updated" => vmm_updated); Ok(()) } - // If the instance doesn't exist, swallow the error -- there's - // nothing to do here. - // TODO-robustness This could only be possible if we've removed an - // Instance from the datastore altogether. When would we do that? - // We don't want to do it as soon as something's destroyed, I think, - // and in that case, we'd need some async task for cleaning these - // up. + // The update command should swallow object-not-found errors and + // return them back as failures to update, so this error case is + // unexpected. There's no work to do if this occurs, however. Err(Error::ObjectNotFound { .. }) => { - warn!(log, "non-existent instance updated by sled agent"; - "instance_id" => %id, - "new_state" => %new_runtime_state.run_state); + error!(log, "instance/vmm update unexpectedly returned \ + an object not found error"; + "instance_id" => %instance_id, + "propolis_id" => %propolis_id); Ok(()) } @@ -1160,83 +1386,28 @@ impl super::Nexus { // different from Error with an Into. Err(error) => { warn!(log, "failed to update instance from sled agent"; - "instance_id" => %id, - "new_state" => %new_runtime_state.run_state, - "error" => ?error); + "instance_id" => %instance_id, + "propolis_id" => %propolis_id, + "error" => ?error); Err(error) } } } - async fn handle_instance_propolis_gen_change( - &self, - opctx: &OpContext, - new_runtime: &nexus::InstanceRuntimeState, - db_instance: &nexus_db_model::Instance, - ) -> Result<(), Error> { - let log = &self.log; - let instance_id = db_instance.id(); - - info!(log, - "updating configuration after Propolis generation change"; - "instance_id" => %instance_id, - "new_sled_id" => %new_runtime.sled_id, - "old_sled_id" => %db_instance.runtime().sled_id); - - // Push updated V2P mappings to all interested sleds. This needs to be - // done irrespective of whether the sled ID actually changed, because - // merely creating the target Propolis on the target sled will create - // XDE devices for its NICs, and creating an XDE device for a virtual IP - // creates a V2P mapping that maps that IP to that sled. This is fine if - // migration succeeded, but if it failed, the instance is running on the - // source sled, and the incorrect mapping needs to be replaced. - // - // TODO(#3107): When XDE no longer creates mappings implicitly, this - // can be restricted to cases where an instance's sled has actually - // changed. - self.create_instance_v2p_mappings( - opctx, - instance_id, - new_runtime.sled_id, - ) - .await?; - - let (.., sled) = LookupPath::new(opctx, &self.db_datastore) - .sled_id(new_runtime.sled_id) - .fetch() - .await?; - - let boundary_switches = - self.boundary_switches(&self.opctx_alloc).await?; - - for switch in &boundary_switches { - let dpd_client = self.dpd_clients.get(switch).ok_or_else(|| { - Error::internal_error(&format!( - "could not find dpd client for {switch}" - )) - })?; - self.instance_ensure_dpd_config( - opctx, - db_instance.id(), - &sled.address(), - None, - dpd_client, - ) - .await?; - } - - Ok(()) - } - /// Returns the requested range of serial console output bytes, /// provided they are still in the propolis-server's cache. pub(crate) async fn instance_serial_console_data( &self, + opctx: &OpContext, instance_lookup: &lookup::Instance<'_>, params: ¶ms::InstanceSerialConsoleRequest, ) -> Result { let client = self - .propolis_client_for_instance(instance_lookup, authz::Action::Read) + .propolis_client_for_instance( + opctx, + instance_lookup, + authz::Action::Read, + ) .await?; let mut request = client.instance_serial_history_get(); if let Some(max_bytes) = params.max_bytes { @@ -1251,10 +1422,12 @@ impl super::Nexus { let data = request .send() .await - .map_err(|_| { - Error::internal_error( - "websocket connection to instance's serial port failed", - ) + .map_err(|e| { + Error::internal_error(&format!( + "websocket connection to instance's serial port failed: \ + {:?}", + e, + )) })? .into_inner(); Ok(params::InstanceSerialConsoleData { @@ -1265,12 +1438,17 @@ impl super::Nexus { pub(crate) async fn instance_serial_console_stream( &self, + opctx: &OpContext, mut client_stream: WebSocketStream, instance_lookup: &lookup::Instance<'_>, params: ¶ms::InstanceSerialConsoleStreamRequest, ) -> Result<(), Error> { let client_addr = match self - .propolis_addr_for_instance(instance_lookup, authz::Action::Modify) + .propolis_addr_for_instance( + opctx, + instance_lookup, + authz::Action::Modify, + ) .await { Ok(x) => x, @@ -1322,48 +1500,64 @@ impl super::Nexus { async fn propolis_addr_for_instance( &self, + opctx: &OpContext, instance_lookup: &lookup::Instance<'_>, action: authz::Action, ) -> Result { - let (.., authz_instance, instance) = - instance_lookup.fetch_for(action).await?; - match instance.runtime_state.state.0 { - InstanceState::Running - | InstanceState::Rebooting - | InstanceState::Migrating - | InstanceState::Repairing => { - let ip_addr = instance - .runtime_state - .propolis_ip - .ok_or_else(|| { - Error::internal_error( - "instance's hypervisor IP address not found", - ) - })? - .ip(); - Ok(SocketAddr::new(ip_addr, PROPOLIS_PORT)) + let (.., authz_instance) = instance_lookup.lookup_for(action).await?; + + let state = self + .db_datastore + .instance_fetch_with_vmm(opctx, &authz_instance) + .await?; + + let (instance, vmm) = (state.instance(), state.vmm()); + if let Some(vmm) = vmm { + match vmm.runtime.state.0 { + InstanceState::Running + | InstanceState::Rebooting + | InstanceState::Migrating + | InstanceState::Repairing => { + Ok(SocketAddr::new(vmm.propolis_ip.ip(), PROPOLIS_PORT)) + } + InstanceState::Creating + | InstanceState::Starting + | InstanceState::Stopping + | InstanceState::Stopped + | InstanceState::Failed => Err(Error::ServiceUnavailable { + internal_message: format!( + "cannot connect to serial console of instance in state \ + {:?}", + vmm.runtime.state.0 + ), + }), + InstanceState::Destroyed => Err(Error::ServiceUnavailable { + internal_message: format!( + "cannot connect to serial console of instance in state \ + {:?}", + InstanceState::Stopped), + }), } - InstanceState::Creating - | InstanceState::Starting - | InstanceState::Stopping - | InstanceState::Stopped - | InstanceState::Failed => Err(Error::ServiceUnavailable { + } else { + Err(Error::ServiceUnavailable { internal_message: format!( - "Cannot connect to hypervisor of instance in state {:?}", - instance.runtime_state.state - ), - }), - InstanceState::Destroyed => Err(authz_instance.not_found()), + "instance is in state {:?} and has no active serial console \ + server", + instance.runtime().nexus_state + ) + }) } } async fn propolis_client_for_instance( &self, + opctx: &OpContext, instance_lookup: &lookup::Instance<'_>, action: authz::Action, ) -> Result { - let client_addr = - self.propolis_addr_for_instance(instance_lookup, action).await?; + let client_addr = self + .propolis_addr_for_instance(opctx, instance_lookup, action) + .await?; Ok(propolis_client::Client::new(&format!("http://{}", client_addr))) } diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs index c383840d38..0f52cbd260 100644 --- a/nexus/src/app/instance_network.rs +++ b/nexus/src/app/instance_network.rs @@ -7,10 +7,12 @@ use crate::app::sagas::retry_until_known_result; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db; use nexus_db_queries::db::identity::Asset; use nexus_db_queries::db::lookup::LookupPath; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; +use omicron_common::api::internal::nexus; use omicron_common::api::internal::shared::SwitchLocation; use sled_agent_client::types::DeleteVirtualNetworkInterfaceHost; use sled_agent_client::types::SetVirtualNetworkInterfaceHost; @@ -474,4 +476,212 @@ impl super::Nexus { Ok(()) } + + /// Deletes an instance's OPTE V2P mappings and the boundary switch NAT + /// entries for its external IPs. + /// + /// This routine returns immediately upon encountering any errors (and will + /// not try to destroy any more objects after the point of failure). + async fn clear_instance_networking_state( + &self, + opctx: &OpContext, + authz_instance: &authz::Instance, + ) -> Result<(), Error> { + self.delete_instance_v2p_mappings(opctx, authz_instance.id()).await?; + + let external_ips = self + .datastore() + .instance_lookup_external_ips(opctx, authz_instance.id()) + .await?; + + let boundary_switches = self.boundary_switches(opctx).await?; + for external_ip in external_ips { + for switch in &boundary_switches { + debug!(&self.log, "deleting instance nat mapping"; + "instance_id" => %authz_instance.id(), + "switch" => switch.to_string(), + "entry" => #?external_ip); + + let dpd_client = + self.dpd_clients.get(switch).ok_or_else(|| { + Error::internal_error(&format!( + "unable to find dendrite client for {switch}" + )) + })?; + + dpd_client + .ensure_nat_entry_deleted( + &self.log, + external_ip.ip, + *external_ip.first_port, + ) + .await + .map_err(|e| { + Error::internal_error(&format!( + "failed to delete nat entry via dpd: {e}" + )) + })?; + } + } + + Ok(()) + } + + /// Given old and new instance runtime states, determines the desired + /// networking configuration for a given instance and ensures it has been + /// propagated to all relevant sleds. + /// + /// # Arguments + /// + /// - opctx: An operation context for this operation. + /// - authz_instance: A resolved authorization context for the instance of + /// interest. + /// - prev_instance_state: The most-recently-recorded instance runtime + /// state for this instance. + /// - new_instance_state: The instance state that the caller of this routine + /// has observed and that should be used to set up this instance's + /// networking state. + /// + /// # Return value + /// + /// `Ok(())` if this routine completed all the operations it wanted to + /// complete, or an appropriate `Err` otherwise. + pub(crate) async fn ensure_updated_instance_network_config( + &self, + opctx: &OpContext, + authz_instance: &authz::Instance, + prev_instance_state: &db::model::InstanceRuntimeState, + new_instance_state: &nexus::InstanceRuntimeState, + ) -> Result<(), Error> { + let log = &self.log; + let instance_id = authz_instance.id(); + + // If this instance update is stale, do nothing, since the superseding + // update may have allowed the instance's location to change further. + if prev_instance_state.gen >= new_instance_state.gen.into() { + debug!(log, + "instance state generation already advanced, \ + won't touch network config"; + "instance_id" => %instance_id); + + return Ok(()); + } + + // If this update will retire the instance's active VMM, delete its + // networking state. It will be re-established the next time the + // instance starts. + if new_instance_state.propolis_id.is_none() { + info!(log, + "instance cleared its Propolis ID, cleaning network config"; + "instance_id" => %instance_id, + "propolis_id" => ?prev_instance_state.propolis_id); + + self.clear_instance_networking_state(opctx, authz_instance).await?; + return Ok(()); + } + + // If the instance still has a migration in progress, don't change + // any networking state until an update arrives that retires that + // migration. + // + // This is needed to avoid the following race: + // + // 1. Migration from S to T completes. + // 2. Migration source sends an update that changes the instance's + // active VMM but leaves the migration ID in place. + // 3. Meanwhile, migration target sends an update that changes the + // instance's active VMM and clears the migration ID. + // 4. The migration target's call updates networking state and commits + // the new instance record. + // 5. The instance migrates from T to T' and Nexus applies networking + // configuration reflecting that the instance is on T'. + // 6. The update in step 2 applies configuration saying the instance + // is on sled T. + if new_instance_state.migration_id.is_some() { + debug!(log, + "instance still has a migration in progress, won't touch \ + network config"; + "instance_id" => %instance_id, + "migration_id" => ?new_instance_state.migration_id); + + return Ok(()); + } + + let new_propolis_id = new_instance_state.propolis_id.unwrap(); + + // Updates that end live migration need to push OPTE V2P state even if + // the instance's active sled did not change (see below). + let migration_retired = prev_instance_state.migration_id.is_some() + && new_instance_state.migration_id.is_none(); + + if (prev_instance_state.propolis_id == new_instance_state.propolis_id) + && !migration_retired + { + debug!(log, "instance didn't move, won't touch network config"; + "instance_id" => %instance_id); + + return Ok(()); + } + + // Either the instance moved from one sled to another, or it attempted + // to migrate and failed. Ensure the correct networking configuration + // exists for its current home. + // + // TODO(#3107) This is necessary even if the instance didn't move, + // because registering a migration target on a sled creates OPTE ports + // for its VNICs, and that creates new V2P mappings on that sled that + // place the relevant virtual IPs on the local sled. Once OPTE stops + // creating these mappings, this path only needs to be taken if an + // instance has changed sleds. + let new_sled_id = match self + .db_datastore + .vmm_fetch(&opctx, authz_instance, &new_propolis_id) + .await + { + Ok(vmm) => vmm.sled_id, + + // A VMM in the active position should never be destroyed. If the + // sled sending this message is the owner of the instance's last + // active VMM and is destroying it, it should also have retired that + // VMM. + Err(Error::ObjectNotFound { .. }) => { + error!(log, "instance's active vmm unexpectedly not found"; + "instance_id" => %instance_id, + "propolis_id" => %new_propolis_id); + + return Ok(()); + } + + Err(e) => return Err(e), + }; + + self.create_instance_v2p_mappings(opctx, instance_id, new_sled_id) + .await?; + + let (.., sled) = LookupPath::new(opctx, &self.db_datastore) + .sled_id(new_sled_id) + .fetch() + .await?; + + let boundary_switches = + self.boundary_switches(&self.opctx_alloc).await?; + + for switch in &boundary_switches { + let dpd_client = self.dpd_clients.get(switch).ok_or_else(|| { + Error::internal_error(&format!( + "could not find dpd client for {switch}" + )) + })?; + self.instance_ensure_dpd_config( + opctx, + instance_id, + &sled.address(), + None, + dpd_client, + ) + .await?; + } + + Ok(()) + } } diff --git a/nexus/src/app/sagas/finalize_disk.rs b/nexus/src/app/sagas/finalize_disk.rs index 859cc5a237..d4f6fc39aa 100644 --- a/nexus/src/app/sagas/finalize_disk.rs +++ b/nexus/src/app/sagas/finalize_disk.rs @@ -79,7 +79,7 @@ impl NexusSaga for SagaFinalizeDisk { silo_id: params.silo_id, project_id: params.project_id, disk_id: params.disk_id, - use_the_pantry: true, + attached_instance_and_sled: None, create_params: params::SnapshotCreate { identity: external::IdentityMetadataCreateParams { name: snapshot_name.clone(), diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs new file mode 100644 index 0000000000..438b92cb84 --- /dev/null +++ b/nexus/src/app/sagas/instance_common.rs @@ -0,0 +1,135 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Common helper functions for instance-related sagas. + +use std::net::{IpAddr, Ipv6Addr}; + +use crate::Nexus; +use chrono::Utc; +use nexus_db_model::{ByteCount, SledReservationConstraints, SledResource}; +use nexus_db_queries::{context::OpContext, db, db::DataStore}; +use omicron_common::api::external::InstanceState; +use steno::ActionError; +use uuid::Uuid; + +/// Reserves resources for a new VMM whose instance has `ncpus` guest logical +/// processors and `guest_memory` bytes of guest RAM. The selected sled is +/// random within the set of sleds allowed by the supplied `constraints`. +/// +/// This function succeeds idempotently if called repeatedly with the same +/// `propolis_id`. +pub async fn reserve_vmm_resources( + nexus: &Nexus, + propolis_id: Uuid, + ncpus: u32, + guest_memory: ByteCount, + constraints: SledReservationConstraints, +) -> Result { + // ALLOCATION POLICY + // + // NOTE: This policy can - and should! - be changed. + // + // See https://rfd.shared.oxide.computer/rfd/0205 for a more complete + // discussion. + // + // Right now, allocate an instance to any random sled agent. This has a few + // problems: + // + // - There's no consideration for "health of the sled" here, other than + // "time_deleted = Null". If the sled is rebooting, in a known unhealthy + // state, etc, we'd currently provision it here. I don't think this is a + // trivial fix, but it's work we'll need to account for eventually. + // + // - This is selecting a random sled from all sleds in the cluster. For + // multi-rack, this is going to fling the sled to an arbitrary system. + // Maybe that's okay, but worth knowing about explicitly. + // + // - This doesn't take into account anti-affinity - users will want to + // schedule instances that belong to a cluster on different failure + // domains. See https://github.com/oxidecomputer/omicron/issues/1705. + let resources = db::model::Resources::new( + ncpus, + ByteCount::try_from(0i64).unwrap(), + guest_memory, + ); + + let resource = nexus + .reserve_on_random_sled( + propolis_id, + nexus_db_model::SledResourceKind::Instance, + resources, + constraints, + ) + .await + .map_err(ActionError::action_failed)?; + + Ok(resource) +} + +/// Creates a new VMM record from the supplied IDs and stores it in the supplied +/// datastore. +/// +/// This function succeeds idempotently if called repeatedly with the same +/// parameters, provided that the VMM record was not mutated by some other actor +/// after the calling saga inserted it. +pub async fn create_and_insert_vmm_record( + datastore: &DataStore, + opctx: &OpContext, + instance_id: Uuid, + propolis_id: Uuid, + sled_id: Uuid, + propolis_ip: Ipv6Addr, + initial_state: nexus_db_model::VmmInitialState, +) -> Result { + let vmm = db::model::Vmm::new( + propolis_id, + instance_id, + sled_id, + IpAddr::V6(propolis_ip).into(), + initial_state, + ); + + let vmm = datastore + .vmm_insert(&opctx, vmm) + .await + .map_err(ActionError::action_failed)?; + + Ok(vmm) +} + +/// Given a previously-inserted VMM record, set its state to Destroyed and then +/// delete it. +/// +/// This function succeeds idempotently if called with the same parameters, +/// provided that the VMM record was not changed by some other actor after the +/// calling saga inserted it. +pub async fn destroy_vmm_record( + datastore: &DataStore, + opctx: &OpContext, + prev_record: &db::model::Vmm, +) -> Result<(), anyhow::Error> { + let new_runtime = db::model::VmmRuntimeState { + state: db::model::InstanceState(InstanceState::Destroyed), + time_state_updated: Utc::now(), + gen: prev_record.runtime.gen.next().into(), + }; + + datastore.vmm_update_runtime(&prev_record.id, &new_runtime).await?; + datastore.vmm_mark_deleted(&opctx, &prev_record.id).await?; + Ok(()) +} + +/// Allocates a new IPv6 address for a service that will run on the supplied +/// sled. +pub(super) async fn allocate_sled_ipv6( + opctx: &OpContext, + datastore: &DataStore, + sled_uuid: Uuid, +) -> Result { + datastore + .next_ipv6_address(opctx, sled_uuid) + .await + .map_err(ActionError::action_failed) +} diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index 2762ecaff3..5d55aaf0fe 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -3,18 +3,14 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::{NexusActionContext, NexusSaga, SagaInitError, ACTION_GENERATE_ID}; -use crate::app::instance::WriteBackUpdatedInstance; use crate::app::sagas::declare_saga_actions; use crate::app::sagas::disk_create::{self, SagaDiskCreate}; -use crate::app::sagas::retry_until_known_result; use crate::app::{ MAX_DISKS_PER_INSTANCE, MAX_EXTERNAL_IPS_PER_INSTANCE, MAX_NICS_PER_INSTANCE, }; use crate::external_api::params; -use chrono::Utc; use nexus_db_model::NetworkInterfaceKind; -use nexus_db_queries::context::OpContext; use nexus_db_queries::db::identity::Resource; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::ByteCount as DbByteCount; @@ -23,20 +19,16 @@ use nexus_db_queries::{authn, authz, db}; use nexus_defaults::DEFAULT_PRIMARY_NIC_NAME; use nexus_types::external_api::params::InstanceDiskAttachment; use omicron_common::api::external::Error; -use omicron_common::api::external::Generation; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::InstanceState; use omicron_common::api::external::Name; -use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::api::internal::shared::SwitchLocation; use serde::Deserialize; use serde::Serialize; -use sled_agent_client::types::InstanceStateRequested; use slog::warn; use std::collections::HashSet; use std::convert::TryFrom; use std::fmt::Debug; -use std::net::Ipv6Addr; use steno::ActionError; use steno::Node; use steno::{DagBuilder, SagaName}; @@ -83,18 +75,11 @@ struct DiskAttachParams { declare_saga_actions! { instance_create; - ALLOC_SERVER -> "server_id" { - + sic_alloc_server - - sic_alloc_server_undo - } VIRTUAL_RESOURCES_ACCOUNT -> "no_result" { + sic_account_virtual_resources - sic_account_virtual_resources_undo } - ALLOC_PROPOLIS_IP -> "propolis_ip" { - + sic_allocate_propolis_ip - } - CREATE_INSTANCE_RECORD -> "instance_name" { + CREATE_INSTANCE_RECORD -> "instance_record" { + sic_create_instance_record - sic_delete_instance_record } @@ -114,23 +99,8 @@ declare_saga_actions! { + sic_attach_disk_to_instance - sic_attach_disk_to_instance_undo } - CONFIGURE_ASIC -> "configure_asic" { - + sic_add_network_config - - sic_remove_network_config - } - V2P_ENSURE_UNDO -> "v2p_ensure_undo" { - + sic_noop - - sic_v2p_ensure_undo - } - V2P_ENSURE -> "v2p_ensure" { - + sic_v2p_ensure - } - INSTANCE_ENSURE_REGISTERED -> "instance_ensure_registered" { - + sic_instance_ensure_registered - - sic_instance_ensure_registered_undo - } - INSTANCE_ENSURE_RUNNING -> "instance_ensure_running" { - + sic_instance_ensure_running + MOVE_TO_STOPPED -> "stopped_instance" { + + sic_move_to_stopped } } @@ -161,15 +131,7 @@ impl NexusSaga for SagaInstanceCreate { })?, )); - builder.append(Node::action( - "propolis_id", - "GeneratePropolisId", - ACTION_GENERATE_ID.as_ref(), - )); - - builder.append(alloc_server_action()); builder.append(virtual_resources_account_action()); - builder.append(alloc_propolis_ip_action()); builder.append(create_instance_record_action()); // Helper function for appending subsagas to our parent saga. @@ -280,7 +242,8 @@ impl NexusSaga for SagaInstanceCreate { )?; } - // Appends the disk create saga as a subsaga directly to the instance create builder. + // Appends the disk create saga as a subsaga directly to the instance + // create builder. for (i, disk) in params.create_params.disks.iter().enumerate() { if let InstanceDiskAttachment::Create(create_disk) = disk { let subsaga_name = @@ -301,8 +264,8 @@ impl NexusSaga for SagaInstanceCreate { } } - // Attaches all disks included in the instance create request, including those which were previously created - // by the disk create subsagas. + // Attaches all disks included in the instance create request, including + // those which were previously created by the disk create subsagas. for (i, disk_attach) in params.create_params.disks.iter().enumerate() { let subsaga_name = SagaName::new(&format!("instance-attach-disk-{i}")); @@ -327,230 +290,11 @@ impl NexusSaga for SagaInstanceCreate { )?; } - // If a primary NIC exists, create a NAT entry for the default external IP, - // as well as additional NAT entries for each requested ephemeral IP - for i in 0..(params.create_params.external_ips.len() + 1) { - for &switch_location in ¶ms.boundary_switches { - let subsaga_name = SagaName::new(&format!( - "instance-configure-nat-{i}-{switch_location}" - )); - let mut subsaga_builder = DagBuilder::new(subsaga_name); - - let basename = format!("ConfigureAsic-{i}-{switch_location}"); - subsaga_builder.append(Node::action( - "configure_asic", - &basename, - CONFIGURE_ASIC.as_ref(), - )); - let net_params = NetworkConfigParams { - saga_params: params.clone(), - instance_id, - which: i, - switch_location, - }; - subsaga_append( - basename, - subsaga_builder.build()?, - &mut builder, - net_params, - i, - )?; - } - } - - // creating instance v2p mappings is not atomic - there are many calls - // to different sled agents that occur. for this to unwind correctly - // given a partial success of the ensure node, the undo node must be - // prior to the ensure node as a separate action. - builder.append(v2p_ensure_undo_action()); - builder.append(v2p_ensure_action()); - - builder.append(instance_ensure_registered_action()); - if params.create_params.start { - builder.append(instance_ensure_running_action()); - } + builder.append(move_to_stopped_action()); Ok(builder.build()?) } } -async fn sic_add_network_config( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let net_params = sagactx.saga_params::()?; - let which = net_params.which; - let instance_id = net_params.instance_id; - let params = net_params.saga_params; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let osagactx = sagactx.user_data(); - let datastore = osagactx.datastore(); - let switch = net_params.switch_location; - let dpd_client = - osagactx.nexus().dpd_clients.get(&switch).ok_or_else(|| { - ActionError::action_failed(Error::internal_error(&format!( - "unable to find client for switch {switch}" - ))) - })?; - - let (.., db_instance) = LookupPath::new(&opctx, &datastore) - .instance_id(instance_id) - .fetch() - .await - .map_err(ActionError::action_failed)?; - - // Read the sled record from the database. This needs to use the instance- - // create context (and not the regular saga context) to leverage its fleet- - // read permissions. - let sled_uuid = db_instance.runtime_state.sled_id; - let (.., sled) = LookupPath::new(&osagactx.nexus().opctx_alloc, &datastore) - .sled_id(sled_uuid) - .fetch() - .await - .map_err(ActionError::action_failed)?; - - // Set up Dendrite configuration using the saga context, which supplies - // access to the instance's device configuration. - osagactx - .nexus() - .instance_ensure_dpd_config( - &opctx, - instance_id, - &sled.address(), - Some(which), - dpd_client, - ) - .await - .map_err(ActionError::action_failed) -} - -async fn sic_remove_network_config( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let net_params = sagactx.saga_params::()?; - let which = net_params.which; - let instance_id = net_params.instance_id; - let switch = net_params.switch_location; - let params = net_params.saga_params; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let osagactx = sagactx.user_data(); - let dpd_client = - osagactx.nexus().dpd_clients.get(&switch).ok_or_else(|| { - Error::internal_error(&format!( - "unable to find client for switch {switch}" - )) - })?; - let datastore = &osagactx.datastore(); - let log = sagactx.user_data().log(); - - debug!(log, "fetching external ip addresses"); - - let target_ip = &datastore - .instance_lookup_external_ips(&opctx, instance_id) - .await - .map_err(ActionError::action_failed)? - .get(which) - .ok_or_else(|| { - ActionError::action_failed(Error::internal_error(&format!( - "failed to find external ip address at index: {which}" - ))) - })? - .to_owned(); - - debug!(log, "deleting nat mapping for entry: {target_ip:#?}"); - - let result = retry_until_known_result(log, || async { - dpd_client - .ensure_nat_entry_deleted(log, target_ip.ip, *target_ip.first_port) - .await - }) - .await; - - match result { - Ok(_) => { - debug!(log, "deletion of nat entry successful for: {target_ip:#?}"); - Ok(()) - } - Err(e) => Err(Error::internal_error(&format!( - "failed to delete nat entry via dpd: {e}" - ))), - }?; - - Ok(()) -} - -async fn sic_alloc_server( - sagactx: NexusActionContext, -) -> Result { - let osagactx = sagactx.user_data(); - - // ALLOCATION POLICY - // - // NOTE: This policy can - and should! - be changed. - // - // See https://rfd.shared.oxide.computer/rfd/0205 for a more complete - // discussion. - // - // Right now, allocate an instance to any random sled agent. This has a few - // problems: - // - // - There's no consideration for "health of the sled" here, other than - // "time_deleted = Null". If the sled is rebooting, in a known unhealthy - // state, etc, we'd currently provision it here. I don't think this is a - // trivial fix, but it's work we'll need to account for eventually. - // - // - This is selecting a random sled from all sleds in the cluster. For - // multi-rack, this is going to fling the sled to an arbitrary system. - // Maybe that's okay, but worth knowing about explicitly. - // - // - This doesn't take into account anti-affinity - users will want to - // schedule instances that belong to a cluster on different failure - // domains. See https://github.com/oxidecomputer/omicron/issues/1705. - - // TODO: Fix these values. They're wrong now, but they let us move - // forward with plumbing. - let params = sagactx.saga_params::()?; - let hardware_threads = params.create_params.ncpus.0; - let rss_ram = params.create_params.memory; - let reservoir_ram = omicron_common::api::external::ByteCount::from(0); - - // Use the instance's Propolis ID as its resource key, since each unique - // Propolis consumes its own resources, and an instance can have multiple - // Propolises during a live migration. - let propolis_id = sagactx.lookup::("propolis_id")?; - let resources = db::model::Resources::new( - hardware_threads.into(), - rss_ram.into(), - reservoir_ram.into(), - ); - - let resource = osagactx - .nexus() - .reserve_on_random_sled( - propolis_id, - db::model::SledResourceKind::Instance, - resources, - db::model::SledReservationConstraints::none(), - ) - .await - .map_err(ActionError::action_failed)?; - Ok(resource.sled_id) -} - -async fn sic_alloc_server_undo( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let osagactx = sagactx.user_data(); - let propolis_id = sagactx.lookup::("propolis_id")?; - - osagactx.nexus().delete_sled_reservation(propolis_id).await?; - Ok(()) -} - /// Create a network interface for an instance, using the parameters at index /// `nic_index`, returning the UUID for the NIC (or None). async fn sic_create_network_interface( @@ -984,24 +728,6 @@ async fn ensure_instance_disk_attach_state( Ok(()) } -/// Helper function to allocate a new IPv6 address for an Oxide service running -/// on the provided sled. -/// -/// `sled_id_name` is the name of the serialized output containing the UUID for -/// the targeted sled. -pub(super) async fn allocate_sled_ipv6( - opctx: &OpContext, - sagactx: NexusActionContext, - sled_uuid: Uuid, -) -> Result { - let osagactx = sagactx.user_data(); - osagactx - .datastore() - .next_ipv6_address(opctx, sled_uuid) - .await - .map_err(ActionError::action_failed) -} - async fn sic_account_virtual_resources( sagactx: NexusActionContext, ) -> Result<(), ActionError> { @@ -1052,56 +778,21 @@ async fn sic_account_virtual_resources_undo( Ok(()) } -// Allocate an IP address on the destination sled for the Propolis server -async fn sic_allocate_propolis_ip( - sagactx: NexusActionContext, -) -> Result { - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let sled_uuid = sagactx.lookup::("server_id")?; - allocate_sled_ipv6(&opctx, sagactx, sled_uuid).await -} - async fn sic_create_instance_record( sagactx: NexusActionContext, -) -> Result { +) -> Result { let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; let opctx = crate::context::op_context_for_saga_action( &sagactx, ¶ms.serialized_authn, ); - let sled_uuid = sagactx.lookup::("server_id")?; let instance_id = sagactx.lookup::("instance_id")?; - let propolis_uuid = sagactx.lookup::("propolis_id")?; - let propolis_addr = sagactx.lookup::("propolis_ip")?; - - let runtime = InstanceRuntimeState { - run_state: InstanceState::Creating, - sled_id: sled_uuid, - propolis_id: propolis_uuid, - dst_propolis_id: None, - propolis_addr: Some(std::net::SocketAddr::new( - propolis_addr.into(), - 12400, - )), - migration_id: None, - propolis_gen: Generation::new(), - hostname: params.create_params.hostname.clone(), - memory: params.create_params.memory, - ncpus: params.create_params.ncpus, - gen: Generation::new(), - time_updated: Utc::now(), - }; let new_instance = db::model::Instance::new( instance_id, params.project_id, ¶ms.create_params, - runtime.into(), ); let (.., authz_project) = LookupPath::new(&opctx, &osagactx.datastore()) @@ -1116,7 +807,7 @@ async fn sic_create_instance_record( .await .map_err(ActionError::action_failed)?; - Ok(instance.name().clone().into()) + Ok(instance) } async fn sic_delete_instance_record( @@ -1130,7 +821,11 @@ async fn sic_delete_instance_record( ¶ms.serialized_authn, ); let instance_id = sagactx.lookup::("instance_id")?; - let instance_name = sagactx.lookup::("instance_name")?; + let instance_name = sagactx + .lookup::("instance_record")? + .name() + .clone() + .into(); // We currently only support deleting an instance if it is stopped or // failed, so update the state accordingly to allow deletion. @@ -1156,7 +851,7 @@ async fn sic_delete_instance_record( }; let runtime_state = db::model::InstanceRuntimeState { - state: db::model::InstanceState::new(InstanceState::Failed), + nexus_state: db::model::InstanceState::new(InstanceState::Failed), // Must update the generation, or the database query will fail. // // The runtime state of the instance record is only changed as a result @@ -1183,186 +878,43 @@ async fn sic_delete_instance_record( Ok(()) } -async fn sic_noop(_sagactx: NexusActionContext) -> Result<(), ActionError> { - Ok(()) -} - -/// Ensure that the necessary v2p mappings exist for this instance -async fn sic_v2p_ensure( +async fn sic_move_to_stopped( sagactx: NexusActionContext, ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let instance_id = sagactx.lookup::("instance_id")?; - let sled_id = sagactx.lookup::("server_id")?; - - osagactx - .nexus() - .create_instance_v2p_mappings(&opctx, instance_id, sled_id) - .await - .map_err(ActionError::action_failed)?; - - Ok(()) -} - -async fn sic_v2p_ensure_undo( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); let instance_id = sagactx.lookup::("instance_id")?; + let instance_record = + sagactx.lookup::("instance_record")?; + + // Create a new generation of the isntance record with the Stopped state and + // try to write it back to the database. If this node is replayed, or the + // instance has already changed state by the time this step is reached, this + // update will (correctly) be ignored because its generation number is out + // of date. + let new_state = db::model::InstanceRuntimeState { + nexus_state: db::model::InstanceState::new(InstanceState::Stopped), + gen: db::model::Generation::from( + instance_record.runtime_state.gen.next(), + ), + ..instance_record.runtime_state + }; - osagactx - .nexus() - .delete_instance_v2p_mappings(&opctx, instance_id) - .await - .map_err(ActionError::action_failed)?; - - Ok(()) -} - -async fn sic_instance_ensure_registered( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let datastore = osagactx.datastore(); - - // TODO-correctness TODO-security It's not correct to re-resolve the - // instance name now. See oxidecomputer/omicron#1536. - let instance_name = sagactx.lookup::("instance_name")?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - - let (.., authz_instance, db_instance) = LookupPath::new(&opctx, &datastore) - .project_id(params.project_id) - .instance_name(&instance_name) - .fetch() + // If this node is being replayed, this instance may already have been + // deleted, so ignore object-not-found errors. + if let Err(e) = osagactx + .datastore() + .instance_update_runtime(&instance_id, &new_state) .await - .map_err(ActionError::action_failed)?; - - if !params.create_params.start { - let instance_id = db_instance.id(); - // If we don't need to start the instance, we can skip the ensure - // and just update the instance runtime state to `Stopped`. - // - // TODO-correctness: This is dangerous if this step is replayed, since - // a user can discover this instance and ask to start it in between - // attempts to run this step. One way to fix this is to avoid refetching - // the previous runtime state each time this step is taken, such that - // once this update is applied once, subsequent attempts to apply it - // will have an already-used generation number. - let runtime_state = db::model::InstanceRuntimeState { - state: db::model::InstanceState::new(InstanceState::Stopped), - // Must update the generation, or the database query will fail. - // - // The runtime state of the instance record is only changed as a - // result of the successful completion of the saga (i.e. after - // ensure which we're skipping in this case) or during saga - // unwinding. So we're guaranteed that the cached generation in the - // saga log is the most recent in the database. - gen: db::model::Generation::from( - db_instance.runtime_state.gen.next(), - ), - ..db_instance.runtime_state - }; - - let updated = datastore - .instance_update_runtime(&instance_id, &runtime_state) - .await - .map_err(ActionError::action_failed)?; - - if !updated { - warn!( - osagactx.log(), - "failed to update instance runtime state from creating to stopped", - ); + { + match e { + Error::ObjectNotFound { .. } => return Ok(()), + e => return Err(ActionError::action_failed(e)), } - } else { - osagactx - .nexus() - .instance_ensure_registered(&opctx, &authz_instance, &db_instance) - .await - .map_err(ActionError::action_failed)?; } Ok(()) } -async fn sic_instance_ensure_registered_undo( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let datastore = osagactx.datastore(); - let instance_id = sagactx.lookup::("instance_id")?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - - let (.., authz_instance, db_instance) = LookupPath::new(&opctx, &datastore) - .instance_id(instance_id) - .fetch() - .await - .map_err(ActionError::action_failed)?; - - osagactx - .nexus() - .instance_ensure_unregistered( - &opctx, - &authz_instance, - &db_instance, - WriteBackUpdatedInstance::WriteBack, - ) - .await - .map_err(ActionError::action_failed)?; - - Ok(()) -} - -async fn sic_instance_ensure_running( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let datastore = osagactx.datastore(); - let instance_id = sagactx.lookup::("instance_id")?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - - let (.., authz_instance, db_instance) = LookupPath::new(&opctx, &datastore) - .instance_id(instance_id) - .fetch() - .await - .map_err(ActionError::action_failed)?; - - osagactx - .nexus() - .instance_request_state( - &opctx, - &authz_instance, - &db_instance, - InstanceStateRequested::Running, - ) - .await - .map_err(ActionError::action_failed)?; - - Ok(()) -} - #[cfg(test)] pub mod test { use crate::{ diff --git a/nexus/src/app/sagas/instance_delete.rs b/nexus/src/app/sagas/instance_delete.rs index 005e9724a6..7da497136e 100644 --- a/nexus/src/app/sagas/instance_delete.rs +++ b/nexus/src/app/sagas/instance_delete.rs @@ -8,9 +8,7 @@ use super::ActionRegistry; use super::NexusActionContext; use super::NexusSaga; use crate::app::sagas::declare_saga_actions; -use nexus_db_queries::db; -use nexus_db_queries::db::lookup::LookupPath; -use nexus_db_queries::{authn, authz}; +use nexus_db_queries::{authn, authz, db}; use nexus_types::identity::Resource; use omicron_common::api::external::{Error, ResourceType}; use omicron_common::api::internal::shared::SwitchLocation; @@ -21,7 +19,7 @@ use steno::ActionError; // instance delete saga: input parameters #[derive(Debug, Deserialize, Serialize)] -pub(crate) struct Params { +pub struct Params { pub serialized_authn: authn::saga::Serialized, pub authz_instance: authz::Instance, pub instance: db::model::Instance, @@ -32,19 +30,10 @@ pub(crate) struct Params { declare_saga_actions! { instance_delete; - V2P_ENSURE_UNDO -> "v2p_ensure_undo" { - + sid_noop - - sid_v2p_ensure_undo - } - V2P_ENSURE -> "v2p_ensure" { - + sid_v2p_ensure - } + INSTANCE_DELETE_RECORD -> "no_result1" { + sid_delete_instance_record } - DELETE_ASIC_CONFIGURATION -> "delete_asic_configuration" { - + sid_delete_network_config - } DELETE_NETWORK_INTERFACES -> "no_result2" { + sid_delete_network_interfaces } @@ -54,15 +43,12 @@ declare_saga_actions! { VIRTUAL_RESOURCES_ACCOUNT -> "no_result4" { + sid_account_virtual_resources } - SLED_RESOURCES_ACCOUNT -> "no_result5" { - + sid_account_sled_resources - } } // instance delete saga: definition #[derive(Debug)] -pub(crate) struct SagaInstanceDelete; +pub struct SagaInstanceDelete; impl NexusSaga for SagaInstanceDelete { const NAME: &'static str = "instance-delete"; type Params = Params; @@ -75,91 +61,16 @@ impl NexusSaga for SagaInstanceDelete { _params: &Self::Params, mut builder: steno::DagBuilder, ) -> Result { - builder.append(v2p_ensure_undo_action()); - builder.append(v2p_ensure_action()); - builder.append(delete_asic_configuration_action()); builder.append(instance_delete_record_action()); builder.append(delete_network_interfaces_action()); builder.append(deallocate_external_ip_action()); builder.append(virtual_resources_account_action()); - builder.append(sled_resources_account_action()); Ok(builder.build()?) } } // instance delete saga: action implementations -async fn sid_noop(_sagactx: NexusActionContext) -> Result<(), ActionError> { - Ok(()) -} - -/// Ensure that the v2p mappings for this instance are deleted -async fn sid_v2p_ensure( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - - osagactx - .nexus() - .delete_instance_v2p_mappings(&opctx, params.authz_instance.id()) - .await - .map_err(ActionError::action_failed)?; - - Ok(()) -} - -/// During unwind, ensure that v2p mappings are created again -async fn sid_v2p_ensure_undo( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - - let (.., db_instance) = LookupPath::new(&opctx, &osagactx.datastore()) - .instance_id(params.authz_instance.id()) - .fetch_for(authz::Action::Read) - .await?; - - osagactx - .nexus() - .create_instance_v2p_mappings( - &opctx, - params.authz_instance.id(), - db_instance.runtime().sled_id, - ) - .await - .map_err(ActionError::action_failed)?; - - Ok(()) -} - -async fn sid_delete_network_config( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let authz_instance = ¶ms.authz_instance; - let osagactx = sagactx.user_data(); - - osagactx - .nexus() - .instance_delete_dpd_config(&opctx, authz_instance) - .await - .map_err(ActionError::action_failed) -} - async fn sid_delete_instance_record( sagactx: NexusActionContext, ) -> Result<(), ActionError> { @@ -240,50 +151,14 @@ async fn sid_account_virtual_resources( &opctx, params.instance.id(), params.instance.project_id, - i64::from(params.instance.runtime_state.ncpus.0 .0), - params.instance.runtime_state.memory, + i64::from(params.instance.ncpus.0 .0), + params.instance.memory, ) .await .map_err(ActionError::action_failed)?; Ok(()) } -async fn sid_account_sled_resources( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - - // Fetch the previously-deleted instance record to get its Propolis ID. It - // is safe to fetch the ID at this point because the instance is already - // deleted and so cannot change anymore. - // - // TODO(#2315): This prevents the garbage collection of soft-deleted - // instance records. A better method is to remove a Propolis's reservation - // once an instance no longer refers to it (e.g. when it has stopped or - // been removed from the instance's migration information) and then make - // this saga check that the instance has no active Propolises before it is - // deleted. This logic should be part of the logic needed to stop an - // instance and release its Propolis reservation; when that is added this - // step can be removed. - let instance = osagactx - .datastore() - .instance_fetch_deleted(&opctx, ¶ms.authz_instance) - .await - .map_err(ActionError::action_failed)?; - - osagactx - .datastore() - .sled_reservation_delete(&opctx, instance.runtime().propolis_id) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - #[cfg(test)] mod test { use crate::{ @@ -415,10 +290,20 @@ mod test { }; let project_lookup = nexus.project_lookup(&opctx, project_selector).unwrap(); - nexus + + let instance_state = nexus .project_create_instance(&opctx, &project_lookup, ¶ms) .await - .unwrap() + .unwrap(); + + let datastore = cptestctx.server.apictx().nexus.datastore().clone(); + let (.., db_instance) = LookupPath::new(&opctx, &datastore) + .instance_id(instance_state.instance().id()) + .fetch() + .await + .expect("test instance should be present in datastore"); + + db_instance } #[nexus_test(server = crate::Server)] diff --git a/nexus/src/app/sagas/instance_migrate.rs b/nexus/src/app/sagas/instance_migrate.rs index 5e9b8680bf..d32a20bc40 100644 --- a/nexus/src/app/sagas/instance_migrate.rs +++ b/nexus/src/app/sagas/instance_migrate.rs @@ -2,23 +2,22 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use super::instance_create::allocate_sled_ipv6; use super::{NexusActionContext, NexusSaga, ACTION_GENERATE_ID}; -use crate::app::instance::WriteBackUpdatedInstance; -use crate::app::sagas::declare_saga_actions; +use crate::app::instance::InstanceStateChangeRequest; +use crate::app::sagas::{ + declare_saga_actions, instance_common::allocate_sled_ipv6, +}; use crate::external_api::params; use nexus_db_queries::db::{identity::Resource, lookup::LookupPath}; use nexus_db_queries::{authn, authz, db}; -use omicron_common::api::external::InstanceState; -use omicron_common::api::internal::nexus::InstanceRuntimeState; +use omicron_common::address::PROPOLIS_PORT; use serde::Deserialize; use serde::Serialize; use sled_agent_client::types::{ InstanceMigrationSourceParams, InstanceMigrationTargetParams, - InstanceStateRequested, }; use slog::warn; -use std::net::Ipv6Addr; +use std::net::{Ipv6Addr, SocketAddr}; use steno::ActionError; use steno::Node; use uuid::Uuid; @@ -26,40 +25,31 @@ use uuid::Uuid; // instance migrate saga: input parameters #[derive(Debug, Deserialize, Serialize)] -pub(crate) struct Params { +pub struct Params { pub serialized_authn: authn::saga::Serialized, pub instance: db::model::Instance, + pub src_vmm: db::model::Vmm, pub migrate_params: params::InstanceMigrate, } -// The migration saga is similar to the instance creation saga: get a -// destination sled, allocate a Propolis process on it, and send it a request to +// The migration saga is similar to the instance start saga: get a destination +// sled, allocate a Propolis process on it, and send that Propolis a request to // initialize via migration, then wait (outside the saga) for this to resolve. -// -// Most of the complexity in this saga comes from the fact that during -// migration, there are two sleds with their own instance runtime states, and -// both the saga and the work that happen after it have to specify carefully -// which of the two participating VMMs is actually running the VM once the -// migration is over. -// -// Only active instances can migrate. While an instance is active on some sled -// (and isn't migrating), that sled's sled agent maintains the instance's -// runtime state and sends updated state to Nexus when it changes. At the start -// of this saga, the participating sled agents and CRDB have the following -// runtime states (note that some fields, like the actual Propolis state, are -// not relevant to migration and are omitted here): -// -// | Item | Source | Dest | CRDB | -// |--------------|--------|------|------| -// | Propolis gen | G | None | G | -// | Propolis ID | P1 | None | P1 | -// | Sled ID | S1 | None | S1 | -// | Dst Prop. ID | None | None | None | -// | Migration ID | None | None | None | + declare_saga_actions! { instance_migrate; - RESERVE_RESOURCES -> "server_id" { + // In order to set up migration, the saga needs to construct the following: + // + // - A migration ID and destination Propolis ID (added to the DAG inline as + // ACTION_GENERATE_ID actions) + // - A sled ID + // - An IP address for the destination Propolis server + // + // The latter two pieces of information are used to create a VMM record for + // the new Propolis, which can then be written into the instance as a + // migration target. + RESERVE_RESOURCES -> "dst_sled_id" { + sim_reserve_sled_resources - sim_release_sled_resources } @@ -68,110 +58,47 @@ declare_saga_actions! { + sim_allocate_propolis_ip } - // This step sets the instance's migration ID and destination Propolis ID + CREATE_VMM_RECORD -> "dst_vmm_record" { + + sim_create_vmm_record + - sim_destroy_vmm_record + } + + // This step the instance's migration ID and destination Propolis ID // fields. Because the instance is active, its current sled agent maintains - // the most recent runtime state, so to update it, the saga calls into the - // sled and asks it to produce an updated record with the appropriate - // migration IDs and a new generation number. + // its most recent runtime state, so to update it, the saga calls into the + // sled and asks it to produce an updated instance record with the + // appropriate migration IDs and a new generation number. // - // Sled agent provides the synchronization here: while this operation is - // idempotent for any single transition between IDs, sled agent ensures that - // if multiple concurrent sagas try to set migration IDs at the same - // Propolis generation, then only one will win and get to proceed through - // the saga. - // - // Once this update completes, the sleds have the following states, and the - // source sled's state will be stored in CRDB: - // - // | Item | Source | Dest | CRDB | - // |--------------|--------|------|------| - // | Propolis gen | G+1 | None | G+1 | - // | Propolis ID | P1 | None | P1 | - // | Sled ID | S1 | None | S1 | - // | Dst Prop. ID | P2 | None | P2 | - // | Migration ID | M | None | M | - // - // Unwinding this step clears the migration IDs using the source sled: - // - // | Item | Source | Dest | CRDB | - // |--------------|--------|------|------| - // | Propolis gen | G+2 | None | G+2 | - // | Propolis ID | P1 | None | P1 | - // | Sled ID | S1 | None | S1 | - // | Dst Prop. ID | None | None | None | - // | Migration ID | None | None | None | + // The source sled agent synchronizes concurrent attempts to set these IDs. + // Setting a new migration ID and re-setting an existing ID are allowed, but + // trying to set an ID when a different ID is already present fails. SET_MIGRATION_IDS -> "set_migration_ids" { + sim_set_migration_ids - sim_clear_migration_ids } - // The instance state on the destination looks like the instance state on - // the source, except that it bears all of the destination's "location" - // information--its Propolis ID, sled ID, and Propolis IP--with the same - // Propolis generation number as the source set in the previous step. - CREATE_DESTINATION_STATE -> "dst_runtime_state" { - + sim_create_destination_state - } - - // Instantiate the new Propolis on the destination sled. This uses the - // record created in the previous step, so the sleds end up with the - // following state: - // - // | Item | Source | Dest | CRDB | - // |--------------|--------|------|------| - // | Propolis gen | G+1 | G+1 | G+1 | - // | Propolis ID | P1 | P2 | P1 | - // | Sled ID | S1 | S2 | S1 | - // | Dst Prop. ID | P2 | P2 | P2 | - // | Migration ID | M | M | M | - // - // Note that, because the source and destination have the same Propolis - // generation, the destination's record will not be written back to CRDB. - // - // Once the migration completes (whether successfully or not), the sled that - // ends up with the instance will publish an update that clears the - // generation numbers and (on success) updates the Propolis ID pointer. If - // migration succeeds, this produces the following: - // - // | Item | Source | Dest | CRDB | - // |--------------|--------|------|------| - // | Propolis gen | G+1 | G+2 | G+2 | - // | Propolis ID | P1 | P2 | P2 | - // | Sled ID | S1 | S2 | S2 | - // | Dst Prop. ID | P2 | None | None | - // | Migration ID | M | None | None | - // - // The undo step for this node requires special care. Unregistering a - // Propolis from a sled typically increments its Propolis generation number. - // (This is so that Nexus can rudely terminate a Propolis via unregistration - // and end up with the state it would have gotten if the Propolis had shut - // down normally.) If this step unwinds, this will produce the same state - // on the destination as in the previous table, even though no migration - // has started yet. If that update gets written back, then it will write - // Propolis generation G+2 to CRDB (as in the table above) with the wrong - // Propolis ID, and the subsequent request to clear migration IDs will not - // fix it (because the source sled's generation number is still at G+1 and - // will move to G+2, which is not recent enough to push another update). - // - // To avoid this problem, this undo step takes special care not to write - // back the updated record the destination sled returns to it. + // This step registers the instance with the destination sled. Care is + // needed at this point because there are two sleds that can send updates + // that affect the same instance record (though they have separate VMMs that + // update independently), and if the saga unwinds they need to ensure they + // cooperate to return the instance to the correct pre-migration state. ENSURE_DESTINATION_PROPOLIS -> "ensure_destination" { + sim_ensure_destination_propolis - sim_ensure_destination_propolis_undo } - // Note that this step only requests migration by sending a "migrate in" - // request to the destination sled. It does not wait for migration to - // finish. It cannot be unwound, either, because there is no way to cancel - // an in-progress migration (indeed, a requested migration might have - // finished entirely by the time the undo step runs). + // Finally, this step requests migration by sending a "migrate in" request + // to the destination sled. It does not wait for migration to finish and + // cannot be allowed to unwind (if a migration has already started, it + // cannot be canceled and indeed may have completed by the time the undo + // step runs). INSTANCE_MIGRATE -> "instance_migrate" { + sim_instance_migrate } } #[derive(Debug)] -pub(crate) struct SagaInstanceMigrate; +pub struct SagaInstanceMigrate; impl NexusSaga for SagaInstanceMigrate { const NAME: &'static str = "instance-migrate"; type Params = Params; @@ -198,8 +125,8 @@ impl NexusSaga for SagaInstanceMigrate { builder.append(reserve_resources_action()); builder.append(allocate_propolis_ip_action()); + builder.append(create_vmm_record_action()); builder.append(set_migration_ids_action()); - builder.append(create_destination_state_action()); builder.append(ensure_destination_propolis_action()); builder.append(instance_migrate_action()); @@ -213,33 +140,23 @@ async fn sim_reserve_sled_resources( ) -> Result { let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; + let propolis_id = sagactx.lookup::("dst_propolis_id")?; - // N.B. This assumes that the instance's shape (CPU/memory allotment) is - // immutable despite being in the instance's "runtime" state. - let resources = db::model::Resources::new( - params.instance.runtime_state.ncpus.0 .0.into(), - params.instance.runtime_state.memory, - // TODO(#2804): Properly specify reservoir size. - omicron_common::api::external::ByteCount::from(0).into(), - ); - - // Add a constraint that the only allowed sled is the one specified in the - // parameters. + // Add a constraint that requires the allocator to reserve on the + // migration's destination sled instead of a random sled. let constraints = db::model::SledReservationConstraintBuilder::new() .must_select_from(&[params.migrate_params.dst_sled_id]) .build(); - let propolis_id = sagactx.lookup::("dst_propolis_id")?; - let resource = osagactx - .nexus() - .reserve_on_random_sled( - propolis_id, - db::model::SledResourceKind::Instance, - resources, - constraints, - ) - .await - .map_err(ActionError::action_failed)?; + let resource = super::instance_common::reserve_vmm_resources( + osagactx.nexus(), + propolis_id, + params.instance.ncpus.0 .0 as u32, + params.instance.memory, + constraints, + ) + .await?; + Ok(resource.sled_id) } @@ -248,6 +165,7 @@ async fn sim_release_sled_resources( ) -> Result<(), anyhow::Error> { let osagactx = sagactx.user_data(); let propolis_id = sagactx.lookup::("dst_propolis_id")?; + osagactx.nexus().delete_sled_reservation(propolis_id).await?; Ok(()) } @@ -261,7 +179,66 @@ async fn sim_allocate_propolis_ip( &sagactx, ¶ms.serialized_authn, ); - allocate_sled_ipv6(&opctx, sagactx, params.migrate_params.dst_sled_id).await + allocate_sled_ipv6( + &opctx, + sagactx.user_data().datastore(), + params.migrate_params.dst_sled_id, + ) + .await +} + +async fn sim_create_vmm_record( + sagactx: NexusActionContext, +) -> Result { + let params = sagactx.saga_params::()?; + let osagactx = sagactx.user_data(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let instance_id = params.instance.id(); + let propolis_id = sagactx.lookup::("dst_propolis_id")?; + let sled_id = sagactx.lookup::("dst_sled_id")?; + let propolis_ip = sagactx.lookup::("dst_propolis_ip")?; + + info!(osagactx.log(), "creating vmm record for migration destination"; + "instance_id" => %instance_id, + "propolis_id" => %propolis_id, + "sled_id" => %sled_id); + + super::instance_common::create_and_insert_vmm_record( + osagactx.datastore(), + &opctx, + instance_id, + propolis_id, + sled_id, + propolis_ip, + nexus_db_model::VmmInitialState::Migrating, + ) + .await +} + +async fn sim_destroy_vmm_record( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let params = sagactx.saga_params::()?; + let osagactx = sagactx.user_data(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let vmm = sagactx.lookup::("dst_vmm_record")?; + info!(osagactx.log(), "destroying vmm record for migration unwind"; + "propolis_id" => %vmm.id); + + super::instance_common::destroy_vmm_record( + osagactx.datastore(), + &opctx, + &vmm, + ) + .await } async fn sim_set_migration_ids( @@ -275,14 +252,24 @@ async fn sim_set_migration_ids( ); let db_instance = ¶ms.instance; + let src_sled_id = params.src_vmm.sled_id; let migration_id = sagactx.lookup::("migrate_id")?; let dst_propolis_id = sagactx.lookup::("dst_propolis_id")?; + + info!(osagactx.log(), "setting migration IDs on migration source sled"; + "instance_id" => %db_instance.id(), + "sled_id" => %src_sled_id, + "migration_id" => %migration_id, + "dst_propolis_id" => %dst_propolis_id, + "prev_runtime_state" => ?db_instance.runtime()); + let updated_record = osagactx .nexus() .instance_set_migration_ids( &opctx, db_instance.id(), - db_instance, + src_sled_id, + db_instance.runtime(), InstanceMigrationSourceParams { dst_propolis_id, migration_id }, ) .await @@ -295,9 +282,16 @@ async fn sim_clear_migration_ids( sagactx: NexusActionContext, ) -> Result<(), anyhow::Error> { let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let src_sled_id = params.src_vmm.sled_id; let db_instance = sagactx.lookup::("set_migration_ids")?; + info!(osagactx.log(), "clearing migration IDs for saga unwind"; + "instance_id" => %db_instance.id(), + "sled_id" => %src_sled_id, + "prev_runtime_state" => ?db_instance.runtime()); + // Because the migration never actually started (and thus didn't finish), // the instance should be at the same Propolis generation as it was when // migration IDs were set, which means sled agent should accept a request to @@ -312,7 +306,11 @@ async fn sim_clear_migration_ids( // as failed. if let Err(e) = osagactx .nexus() - .instance_clear_migration_ids(db_instance.id(), &db_instance) + .instance_clear_migration_ids( + db_instance.id(), + src_sled_id, + db_instance.runtime(), + ) .await { warn!(osagactx.log(), @@ -324,28 +322,6 @@ async fn sim_clear_migration_ids( Ok(()) } -async fn sim_create_destination_state( - sagactx: NexusActionContext, -) -> Result { - let params = sagactx.saga_params::()?; - let mut db_instance = - sagactx.lookup::("set_migration_ids")?; - let dst_propolis_id = sagactx.lookup::("dst_propolis_id")?; - let dst_propolis_ip = sagactx.lookup::("dst_propolis_ip")?; - - // Update the runtime state to refer to the new Propolis. - let new_runtime = db::model::InstanceRuntimeState { - state: db::model::InstanceState::new(InstanceState::Creating), - sled_id: params.migrate_params.dst_sled_id, - propolis_id: dst_propolis_id, - propolis_ip: Some(ipnetwork::Ipv6Network::from(dst_propolis_ip).into()), - ..db_instance.runtime_state - }; - - db_instance.runtime_state = new_runtime; - Ok(db_instance) -} - async fn sim_ensure_destination_propolis( sagactx: NexusActionContext, ) -> Result<(), ActionError> { @@ -355,8 +331,16 @@ async fn sim_ensure_destination_propolis( &sagactx, ¶ms.serialized_authn, ); + + let vmm = sagactx.lookup::("dst_vmm_record")?; let db_instance = - sagactx.lookup::("dst_runtime_state")?; + sagactx.lookup::("set_migration_ids")?; + + info!(osagactx.log(), "ensuring migration destination vmm exists"; + "instance_id" => %db_instance.id(), + "dst_propolis_id" => %vmm.id, + "dst_vmm_state" => ?vmm); + let (.., authz_instance) = LookupPath::new(&opctx, &osagactx.datastore()) .instance_id(db_instance.id()) .lookup_for(authz::Action::Modify) @@ -365,7 +349,13 @@ async fn sim_ensure_destination_propolis( osagactx .nexus() - .instance_ensure_registered(&opctx, &authz_instance, &db_instance) + .instance_ensure_registered( + &opctx, + &authz_instance, + &db_instance, + &vmm.id, + &vmm, + ) .await .map_err(ActionError::action_failed)?; @@ -381,27 +371,39 @@ async fn sim_ensure_destination_propolis_undo( &sagactx, ¶ms.serialized_authn, ); + + let dst_sled_id = sagactx.lookup::("dst_sled_id")?; let db_instance = - sagactx.lookup::("dst_runtime_state")?; + sagactx.lookup::("set_migration_ids")?; let (.., authz_instance) = LookupPath::new(&opctx, &osagactx.datastore()) .instance_id(db_instance.id()) .lookup_for(authz::Action::Modify) .await .map_err(ActionError::action_failed)?; + info!(osagactx.log(), "unregistering destination vmm for migration unwind"; + "instance_id" => %db_instance.id(), + "sled_id" => %dst_sled_id, + "prev_runtime_state" => ?db_instance.runtime()); + // Ensure that the destination sled has no Propolis matching the description // the saga previously generated. // - // The updated instance record from this undo action must be dropped so - // that a later undo action (clearing migration IDs) can update the record - // instead. See the saga definition for more details. + // Sled agent guarantees that if an instance is unregistered from a sled + // that does not believe it holds the "active" Propolis for the instance, + // then the sled's copy of the instance record will not change during + // unregistration. This precondition always holds here because the "start + // migration" step is not allowed to unwind once migration has possibly + // started. Not changing the instance is important here because the next + // undo step (clearing migration IDs) needs to advance the instance's + // generation number to succeed. osagactx .nexus() .instance_ensure_unregistered( &opctx, &authz_instance, - &db_instance, - WriteBackUpdatedInstance::Drop, + &dst_sled_id, + db_instance.runtime(), ) .await .map_err(ActionError::action_failed)?; @@ -418,19 +420,26 @@ async fn sim_instance_migrate( &sagactx, ¶ms.serialized_authn, ); - let src_runtime: InstanceRuntimeState = sagactx - .lookup::("set_migration_ids")? - .runtime() - .clone() - .into(); - let dst_db_instance = - sagactx.lookup::("dst_runtime_state")?; + + let db_instance = + sagactx.lookup::("set_migration_ids")?; + + let src_vmm_addr = + SocketAddr::new(params.src_vmm.propolis_ip.ip(), PROPOLIS_PORT); + + let src_propolis_id = db_instance.runtime().propolis_id.unwrap(); + let dst_vmm = sagactx.lookup::("dst_vmm_record")?; let (.., authz_instance) = LookupPath::new(&opctx, &osagactx.datastore()) - .instance_id(dst_db_instance.id()) + .instance_id(db_instance.id()) .lookup_for(authz::Action::Modify) .await .map_err(ActionError::action_failed)?; + info!(osagactx.log(), "initiating migration from destination sled"; + "instance_id" => %db_instance.id(), + "dst_vmm_record" => ?dst_vmm, + "src_propolis_id" => %src_propolis_id); + // TODO-correctness: This needs to be retried if a transient error occurs to // avoid a problem like the following: // @@ -450,14 +459,12 @@ async fn sim_instance_migrate( .instance_request_state( &opctx, &authz_instance, - &dst_db_instance, - InstanceStateRequested::MigrationTarget( + &db_instance, + &Some(dst_vmm), + InstanceStateChangeRequest::Migrate( InstanceMigrationTargetParams { - src_propolis_addr: src_runtime - .propolis_addr - .unwrap() - .to_string(), - src_propolis_id: src_runtime.propolis_id, + src_propolis_addr: src_vmm_addr.to_string(), + src_propolis_id, }, ), ) @@ -552,26 +559,8 @@ mod tests { .await } - async fn fetch_db_instance( - cptestctx: &ControlPlaneTestContext, - opctx: &nexus_db_queries::context::OpContext, - id: Uuid, - ) -> nexus_db_model::Instance { - let datastore = cptestctx.server.apictx().nexus.datastore().clone(); - let (.., db_instance) = LookupPath::new(&opctx, &datastore) - .instance_id(id) - .fetch() - .await - .expect("test instance should be present in datastore"); - - info!(&cptestctx.logctx.log, "refetched instance from db"; - "instance" => ?db_instance); - - db_instance - } - fn select_first_alternate_sled( - db_instance: &db::model::Instance, + db_vmm: &db::model::Vmm, other_sleds: &[(Uuid, Server)], ) -> Uuid { let default_sled_uuid = @@ -584,7 +573,7 @@ mod tests { panic!("default test sled agent was in other_sleds"); } - if db_instance.runtime().sled_id == default_sled_uuid { + if db_vmm.sled_id == default_sled_uuid { other_sleds[0].0 } else { default_sled_uuid @@ -606,14 +595,14 @@ mod tests { // Poke the instance to get it into the Running state. test_helpers::instance_simulate(cptestctx, &instance.identity.id).await; - let db_instance = - fetch_db_instance(cptestctx, &opctx, instance.identity.id).await; - let old_runtime = db_instance.runtime().clone(); - let dst_sled_id = - select_first_alternate_sled(&db_instance, &other_sleds); + let state = + test_helpers::instance_fetch(cptestctx, instance.identity.id).await; + let vmm = state.vmm().as_ref().unwrap(); + let dst_sled_id = select_first_alternate_sled(vmm, &other_sleds); let params = Params { serialized_authn: authn::saga::Serialized::for_opctx(&opctx), - instance: db_instance, + instance: state.instance().clone(), + src_vmm: vmm.clone(), migrate_params: params::InstanceMigrate { dst_sled_id }, }; @@ -624,12 +613,13 @@ mod tests { // Merely running the migration saga (without simulating any completion // steps in the simulated agents) should not change where the instance // is running. - let new_db_instance = - fetch_db_instance(cptestctx, &opctx, instance.identity.id).await; - assert_eq!(new_db_instance.runtime().sled_id, old_runtime.sled_id); + let new_state = + test_helpers::instance_fetch(cptestctx, state.instance().id()) + .await; + assert_eq!( - new_db_instance.runtime().propolis_id, - old_runtime.propolis_id + new_state.instance().runtime().propolis_id, + state.instance().runtime().propolis_id ); } @@ -649,26 +639,35 @@ mod tests { // Poke the instance to get it into the Running state. test_helpers::instance_simulate(cptestctx, &instance.identity.id).await; - let db_instance = - fetch_db_instance(cptestctx, &opctx, instance.identity.id).await; - let old_runtime = db_instance.runtime().clone(); - let dst_sled_id = - select_first_alternate_sled(&db_instance, &other_sleds); - let make_params = || -> futures::future::BoxFuture<'_, Params> { Box::pin({ async { - let db_instance = fetch_db_instance( + let old_state = test_helpers::instance_fetch( cptestctx, - &opctx, instance.identity.id, ) .await; + + let old_instance = old_state.instance(); + let old_vmm = old_state + .vmm() + .as_ref() + .expect("instance should have a vmm before migrating"); + + let dst_sled_id = + select_first_alternate_sled(old_vmm, &other_sleds); + + info!(log, "setting up new migration saga"; + "old_instance" => ?old_instance, + "src_vmm" => ?old_vmm, + "dst_sled_id" => %dst_sled_id); + Params { serialized_authn: authn::saga::Serialized::for_opctx( &opctx, ), - instance: db_instance, + instance: old_instance.clone(), + src_vmm: old_vmm.clone(), migrate_params: params::InstanceMigrate { dst_sled_id }, } } @@ -681,25 +680,27 @@ mod tests { // Unwinding at any step should clear the migration IDs from // the instance record and leave the instance's location // otherwise untouched. - let new_db_instance = fetch_db_instance( + let new_state = test_helpers::instance_fetch( cptestctx, - &opctx, instance.identity.id, ) .await; - assert!(new_db_instance.runtime().migration_id.is_none()); - assert!(new_db_instance - .runtime() - .dst_propolis_id - .is_none()); + let new_instance = new_state.instance(); + let new_vmm = + new_state.vmm().as_ref().expect("vmm should be active"); + + assert!(new_instance.runtime().migration_id.is_none()); + assert!(new_instance.runtime().dst_propolis_id.is_none()); assert_eq!( - new_db_instance.runtime().sled_id, - old_runtime.sled_id + new_instance.runtime().propolis_id.unwrap(), + new_vmm.id ); - assert_eq!( - new_db_instance.runtime().propolis_id, - old_runtime.propolis_id + + info!( + &log, + "migration saga unwind: stopping instance after failed \ + saga" ); // Ensure the instance can stop. This helps to check that @@ -716,18 +717,28 @@ mod tests { &instance.identity.id, ) .await; - let new_db_instance = fetch_db_instance( + + let new_state = test_helpers::instance_fetch( cptestctx, - &opctx, instance.identity.id, ) .await; + + let new_instance = new_state.instance(); + let new_vmm = new_state.vmm().as_ref(); assert_eq!( - new_db_instance.runtime().state.0, - InstanceState::Stopped + new_instance.runtime().nexus_state.0, + omicron_common::api::external::InstanceState::Stopped ); + assert!(new_instance.runtime().propolis_id.is_none()); + assert!(new_vmm.is_none()); // Restart the instance for the next iteration. + info!( + &log, + "migration saga unwind: restarting instance after \ + failed saga" + ); test_helpers::instance_start( cptestctx, &instance.identity.id, diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index 68e88b0d13..5d02d44b6b 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -4,39 +4,50 @@ //! Implements a saga that starts an instance. -use super::{NexusActionContext, NexusSaga, SagaInitError}; -use crate::app::{ - instance::WriteBackUpdatedInstance, - sagas::{declare_saga_actions, retry_until_known_result}, +use std::net::Ipv6Addr; + +use super::{ + instance_common::allocate_sled_ipv6, NexusActionContext, NexusSaga, + SagaInitError, ACTION_GENERATE_ID, }; +use crate::app::sagas::declare_saga_actions; +use chrono::Utc; use nexus_db_queries::db::{identity::Resource, lookup::LookupPath}; use nexus_db_queries::{authn, authz, db}; use omicron_common::api::external::{Error, InstanceState}; use serde::{Deserialize, Serialize}; -use sled_agent_client::types::InstanceStateRequested; use slog::info; -use steno::ActionError; +use steno::{ActionError, Node}; +use uuid::Uuid; /// Parameters to the instance start saga. #[derive(Debug, Deserialize, Serialize)] pub(crate) struct Params { - pub instance: db::model::Instance, + pub db_instance: db::model::Instance, /// Authentication context to use to fetch the instance's current state from /// the database. pub serialized_authn: authn::saga::Serialized, - - /// True if the saga should configure Dendrite and OPTE configuration for - /// this instance. This allows the instance create saga to do this work - /// prior to invoking the instance start saga as a subsaga without repeating - /// these steps. - pub ensure_network: bool, } declare_saga_actions! { instance_start; - MARK_AS_STARTING -> "starting_state" { + ALLOC_SERVER -> "sled_id" { + + sis_alloc_server + - sis_alloc_server_undo + } + + ALLOC_PROPOLIS_IP -> "propolis_ip" { + + sis_alloc_propolis_ip + } + + CREATE_VMM_RECORD -> "vmm_record" { + + sis_create_vmm_record + - sis_destroy_vmm_record + } + + MARK_AS_STARTING -> "started_record" { + sis_move_to_starting - sis_move_to_starting_undo } @@ -77,6 +88,15 @@ impl NexusSaga for SagaInstanceStart { _params: &Self::Params, mut builder: steno::DagBuilder, ) -> Result { + builder.append(Node::action( + "propolis_id", + "GeneratePropolisId", + ACTION_GENERATE_ID.as_ref(), + )); + + builder.append(alloc_server_action()); + builder.append(alloc_propolis_ip_action()); + builder.append(create_vmm_record_action()); builder.append(mark_as_starting_action()); builder.append(dpd_ensure_action()); builder.append(v2p_ensure_action()); @@ -86,118 +106,200 @@ impl NexusSaga for SagaInstanceStart { } } +async fn sis_alloc_server( + sagactx: NexusActionContext, +) -> Result { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let hardware_threads = params.db_instance.ncpus.0; + let reservoir_ram = params.db_instance.memory; + let propolis_id = sagactx.lookup::("propolis_id")?; + + let resource = super::instance_common::reserve_vmm_resources( + osagactx.nexus(), + propolis_id, + hardware_threads.0 as u32, + reservoir_ram, + db::model::SledReservationConstraints::none(), + ) + .await?; + + Ok(resource.sled_id) +} + +async fn sis_alloc_server_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let propolis_id = sagactx.lookup::("propolis_id")?; + + osagactx.nexus().delete_sled_reservation(propolis_id).await?; + Ok(()) +} + +async fn sis_alloc_propolis_ip( + sagactx: NexusActionContext, +) -> Result { + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let sled_uuid = sagactx.lookup::("sled_id")?; + allocate_sled_ipv6(&opctx, sagactx.user_data().datastore(), sled_uuid).await +} + +async fn sis_create_vmm_record( + sagactx: NexusActionContext, +) -> Result { + let params = sagactx.saga_params::()?; + let osagactx = sagactx.user_data(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let instance_id = params.db_instance.id(); + let propolis_id = sagactx.lookup::("propolis_id")?; + let sled_id = sagactx.lookup::("sled_id")?; + let propolis_ip = sagactx.lookup::("propolis_ip")?; + + super::instance_common::create_and_insert_vmm_record( + osagactx.datastore(), + &opctx, + instance_id, + propolis_id, + sled_id, + propolis_ip, + nexus_db_model::VmmInitialState::Starting, + ) + .await +} + +async fn sis_destroy_vmm_record( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let params = sagactx.saga_params::()?; + let osagactx = sagactx.user_data(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let vmm = sagactx.lookup::("vmm_record")?; + super::instance_common::destroy_vmm_record( + osagactx.datastore(), + &opctx, + &vmm, + ) + .await +} + async fn sis_move_to_starting( sagactx: NexusActionContext, -) -> Result { +) -> Result { let params = sagactx.saga_params::()?; let osagactx = sagactx.user_data(); - let instance_id = params.instance.id(); + let datastore = osagactx.datastore(); + let instance_id = params.db_instance.id(); + let propolis_id = sagactx.lookup::("propolis_id")?; info!(osagactx.log(), "moving instance to Starting state via saga"; - "instance_id" => %instance_id); + "instance_id" => %instance_id, + "propolis_id" => %propolis_id); let opctx = crate::context::op_context_for_saga_action( &sagactx, ¶ms.serialized_authn, ); - // The saga invoker needs to supply a prior state in which the instance can - // legally be started. This action will try to transition the instance to - // the Starting state; once this succeeds, the instance can't be deleted, so - // it is safe to program its network configuration (if required) and then - // try to start it. - // - // This interlock is not sufficient to handle multiple concurrent instance - // creation sagas. See below. - if !matches!( - params.instance.runtime_state.state.0, - InstanceState::Creating | InstanceState::Stopped, - ) { - return Err(ActionError::action_failed(Error::conflict(&format!( - "instance is in state {}, but must be one of {} or {} to be started", - params.instance.runtime_state.state.0, - InstanceState::Creating, - InstanceState::Stopped - )))); - } + // For idempotency, refetch the instance to see if this step already applied + // its desired update. + let (.., db_instance) = LookupPath::new(&opctx, &datastore) + .instance_id(instance_id) + .fetch_for(authz::Action::Modify) + .await + .map_err(ActionError::action_failed)?; - let new_runtime = db::model::InstanceRuntimeState { - state: db::model::InstanceState::new(InstanceState::Starting), - gen: params.instance.runtime_state.gen.next().into(), - ..params.instance.runtime_state - }; + match db_instance.runtime().propolis_id { + // If this saga's Propolis ID is already written to the record, then + // this step must have completed already and is being retried, so + // proceed. + Some(db_id) if db_id == propolis_id => { + info!(osagactx.log(), "start saga: Propolis ID already set"; + "instance_id" => %instance_id); - if !osagactx - .datastore() - .instance_update_runtime(&instance_id, &new_runtime) - .await - .map_err(ActionError::action_failed)? - { - // If the update was not applied, but the desired state is already - // what's in the database, proceed anyway. - // - // TODO(#2315) This logic is not completely correct. It provides - // idempotency in the case where this action moved the instance to - // Starting, but the action was then replayed. It does not handle the - // case where the conflict occurred because a different instance of this - // saga won the race to set the instance to Starting; this will lead to - // two sagas concurrently trying to start the instance. - // - // The correct way to handle this case is to use saga-generated Propolis - // IDs to distinguish between saga executions: the ID must be NULL in - // order to start the instance; if multiple saga executions race, only - // one will write its chosen ID to the record, allowing the sagas to - // determine a winner. - let (.., new_instance) = LookupPath::new(&opctx, &osagactx.datastore()) - .instance_id(instance_id) - .fetch() - .await - .map_err(ActionError::action_failed)?; + Ok(db_instance) + } - if new_instance.runtime_state.gen != new_runtime.gen - || !matches!( - new_instance.runtime_state.state.0, - InstanceState::Starting - ) - { + // If the instance has a different Propolis ID, a competing start saga + // must have started the instance already, so unwind. + Some(_) => { return Err(ActionError::action_failed(Error::conflict( "instance changed state before it could be started", ))); } - info!(osagactx.log(), "start saga: instance was already starting"; - "instance_id" => %instance_id); + // If the instance has no Propolis ID, try to write this saga's chosen + // ID into the instance and put it in the Running state. (While the + // instance is still technically starting up, writing the Propolis ID at + // this point causes the VMM's state, which is Starting, to supersede + // the instance's state, so this won't cause the instance to appear to + // be running before Propolis thinks it has started.) + None => { + let new_runtime = db::model::InstanceRuntimeState { + nexus_state: db::model::InstanceState::new( + InstanceState::Running, + ), + propolis_id: Some(propolis_id), + time_updated: Utc::now(), + gen: db_instance.runtime().gen.next().into(), + ..db_instance.runtime_state + }; + + // Bail if another actor managed to update the instance's state in + // the meantime. + if !osagactx + .datastore() + .instance_update_runtime(&instance_id, &new_runtime) + .await + .map_err(ActionError::action_failed)? + { + return Err(ActionError::action_failed(Error::conflict( + "instance changed state before it could be started", + ))); + } + + let mut new_record = db_instance.clone(); + new_record.runtime_state = new_runtime; + Ok(new_record) + } } - - Ok(new_runtime) } async fn sis_move_to_starting_undo( sagactx: NexusActionContext, ) -> Result<(), anyhow::Error> { - let params = sagactx.saga_params::()?; let osagactx = sagactx.user_data(); + let db_instance = + sagactx.lookup::("started_record")?; + let instance_id = db_instance.id(); info!(osagactx.log(), "start saga failed; returning instance to Stopped"; - "instance_id" => %params.instance.id()); - - let runtime_state = - sagactx.lookup::("starting_state")?; + "instance_id" => %instance_id); - // Don't just restore the old state; if the instance was being created, and - // starting it failed, the instance is now stopped, not creating. let new_runtime = db::model::InstanceRuntimeState { - state: db::model::InstanceState::new(InstanceState::Stopped), - gen: runtime_state.gen.next().into(), - ..runtime_state + nexus_state: db::model::InstanceState::new(InstanceState::Stopped), + propolis_id: None, + gen: db_instance.runtime_state.gen.next().into(), + ..db_instance.runtime_state }; if !osagactx .datastore() - .instance_update_runtime(¶ms.instance.id(), &new_runtime) + .instance_update_runtime(&instance_id, &new_runtime) .await? { info!(osagactx.log(), "did not return instance to Stopped: old generation number"; - "instance_id" => %params.instance.id()); + "instance_id" => %instance_id); } Ok(()) @@ -208,25 +310,20 @@ async fn sis_dpd_ensure( ) -> Result<(), ActionError> { let params = sagactx.saga_params::()?; let osagactx = sagactx.user_data(); - if !params.ensure_network { - info!(osagactx.log(), "start saga: skipping dpd_ensure by request"; - "instance_id" => %params.instance.id()); - - return Ok(()); - } + let db_instance = + sagactx.lookup::("started_record")?; + let instance_id = db_instance.id(); info!(osagactx.log(), "start saga: ensuring instance dpd configuration"; - "instance_id" => %params.instance.id()); + "instance_id" => %instance_id); let opctx = crate::context::op_context_for_saga_action( &sagactx, ¶ms.serialized_authn, ); let datastore = osagactx.datastore(); - let runtime_state = - sagactx.lookup::("starting_state")?; - let sled_uuid = runtime_state.sled_id; + let sled_uuid = sagactx.lookup::("sled_id")?; let (.., sled) = LookupPath::new(&osagactx.nexus().opctx_alloc, &datastore) .sled_id(sled_uuid) .fetch() @@ -251,7 +348,7 @@ async fn sis_dpd_ensure( .nexus() .instance_ensure_dpd_config( &opctx, - params.instance.id(), + instance_id, &sled.address(), None, dpd_client, @@ -267,57 +364,27 @@ async fn sis_dpd_ensure_undo( sagactx: NexusActionContext, ) -> Result<(), anyhow::Error> { let params = sagactx.saga_params::()?; + let instance_id = params.db_instance.id(); let osagactx = sagactx.user_data(); let log = osagactx.log(); - if !params.ensure_network { - info!(log, - "start saga: didn't ensure dpd configuration, nothing to undo"; - "instance_id" => %params.instance.id()); - - return Ok(()); - } - - info!(log, "start saga: undoing dpd configuration"; - "instance_id" => %params.instance.id()); - - let datastore = &osagactx.datastore(); let opctx = crate::context::op_context_for_saga_action( &sagactx, ¶ms.serialized_authn, ); - let target_ips = &datastore - .instance_lookup_external_ips(&opctx, params.instance.id()) - .await?; + info!(log, "start saga: undoing dpd configuration"; + "instance_id" => %instance_id); - let boundary_switches = osagactx.nexus().boundary_switches(&opctx).await?; - for switch in boundary_switches { - let dpd_client = - osagactx.nexus().dpd_clients.get(&switch).ok_or_else(|| { - ActionError::action_failed(Error::internal_error(&format!( - "unable to find client for switch {switch}" - ))) - })?; + let (.., authz_instance) = LookupPath::new(&opctx, &osagactx.datastore()) + .instance_id(instance_id) + .lookup_for(authz::Action::Modify) + .await + .map_err(ActionError::action_failed)?; - for ip in target_ips { - let result = retry_until_known_result(log, || async { - dpd_client - .ensure_nat_entry_deleted(log, ip.ip, *ip.first_port) - .await - }) - .await; - - match result { - Ok(_) => { - debug!(log, "successfully deleted nat entry for {ip:#?}"); - Ok(()) - } - Err(e) => Err(Error::internal_error(&format!( - "failed to delete nat entry for {ip:#?} via dpd: {e}" - ))), - }?; - } - } + osagactx + .nexus() + .instance_delete_dpd_config(&opctx, &authz_instance) + .await?; Ok(()) } @@ -327,27 +394,20 @@ async fn sis_v2p_ensure( ) -> Result<(), ActionError> { let params = sagactx.saga_params::()?; let osagactx = sagactx.user_data(); - if !params.ensure_network { - info!(osagactx.log(), "start saga: skipping v2p_ensure by request"; - "instance_id" => %params.instance.id()); - - return Ok(()); - } + let instance_id = params.db_instance.id(); info!(osagactx.log(), "start saga: ensuring v2p mappings are configured"; - "instance_id" => %params.instance.id()); + "instance_id" => %instance_id); let opctx = crate::context::op_context_for_saga_action( &sagactx, ¶ms.serialized_authn, ); - let runtime_state = - sagactx.lookup::("starting_state")?; - let sled_uuid = runtime_state.sled_id; + let sled_uuid = sagactx.lookup::("sled_id")?; osagactx .nexus() - .create_instance_v2p_mappings(&opctx, params.instance.id(), sled_uuid) + .create_instance_v2p_mappings(&opctx, instance_id, sled_uuid) .await .map_err(ActionError::action_failed)?; @@ -359,17 +419,11 @@ async fn sis_v2p_ensure_undo( ) -> Result<(), anyhow::Error> { let params = sagactx.saga_params::()?; let osagactx = sagactx.user_data(); - if !params.ensure_network { - info!(osagactx.log(), - "start saga: didn't ensure v2p configuration, nothing to undo"; - "instance_id" => %params.instance.id()); - - return Ok(()); - } - - let instance_id = params.instance.id(); + let instance_id = params.db_instance.id(); + let sled_id = sagactx.lookup::("sled_id")?; info!(osagactx.log(), "start saga: undoing v2p configuration"; - "instance_id" => %instance_id); + "instance_id" => %instance_id, + "sled_id" => %sled_id); let opctx = crate::context::op_context_for_saga_action( &sagactx, @@ -394,29 +448,32 @@ async fn sis_ensure_registered( ¶ms.serialized_authn, ); let osagactx = sagactx.user_data(); + let db_instance = + sagactx.lookup::("started_record")?; + let instance_id = db_instance.id(); + let sled_id = sagactx.lookup::("sled_id")?; + let vmm_record = sagactx.lookup::("vmm_record")?; + let propolis_id = sagactx.lookup::("propolis_id")?; info!(osagactx.log(), "start saga: ensuring instance is registered on sled"; - "instance_id" => %params.instance.id(), - "sled_id" => %params.instance.runtime().sled_id); - - let (.., authz_instance, mut db_instance) = - LookupPath::new(&opctx, &osagactx.datastore()) - .instance_id(params.instance.id()) - .fetch_for(authz::Action::Modify) - .await - .map_err(ActionError::action_failed)?; + "instance_id" => %instance_id, + "sled_id" => %sled_id); - // The instance is not really being "created" (it already exists from - // the caller's perspective), but if it does not exist on its sled, the - // target sled agent will populate its instance manager with the - // contents of this modified record, and that record needs to allow a - // transition to the Starting state. - db_instance.runtime_state.state = - nexus_db_model::InstanceState(InstanceState::Creating); + let (.., authz_instance) = LookupPath::new(&opctx, &osagactx.datastore()) + .instance_id(instance_id) + .lookup_for(authz::Action::Modify) + .await + .map_err(ActionError::action_failed)?; osagactx .nexus() - .instance_ensure_registered(&opctx, &authz_instance, &db_instance) + .instance_ensure_registered( + &opctx, + &authz_instance, + &db_instance, + &propolis_id, + &vmm_record, + ) .await .map_err(ActionError::action_failed)?; @@ -429,7 +486,8 @@ async fn sis_ensure_registered_undo( let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; let datastore = osagactx.datastore(); - let instance_id = params.instance.id(); + let instance_id = params.db_instance.id(); + let sled_id = sagactx.lookup::("sled_id")?; let opctx = crate::context::op_context_for_saga_action( &sagactx, ¶ms.serialized_authn, @@ -437,8 +495,10 @@ async fn sis_ensure_registered_undo( info!(osagactx.log(), "start saga: unregistering instance from sled"; "instance_id" => %instance_id, - "sled_id" => %params.instance.runtime().sled_id); + "sled_id" => %sled_id); + // Fetch the latest record so that this callee can drive the instance into + // a Failed state if the unregister call fails. let (.., authz_instance, db_instance) = LookupPath::new(&opctx, &datastore) .instance_id(instance_id) .fetch() @@ -450,8 +510,8 @@ async fn sis_ensure_registered_undo( .instance_ensure_unregistered( &opctx, &authz_instance, - &db_instance, - WriteBackUpdatedInstance::WriteBack, + &sled_id, + db_instance.runtime(), ) .await .map_err(ActionError::action_failed)?; @@ -470,12 +530,18 @@ async fn sis_ensure_running( ¶ms.serialized_authn, ); + let db_instance = + sagactx.lookup::("started_record")?; + let db_vmm = sagactx.lookup::("vmm_record")?; + let instance_id = params.db_instance.id(); + let sled_id = sagactx.lookup::("sled_id")?; info!(osagactx.log(), "start saga: ensuring instance is running"; - "instance_id" => %params.instance.id()); + "instance_id" => %instance_id, + "sled_id" => %sled_id); - let (.., authz_instance, db_instance) = LookupPath::new(&opctx, &datastore) - .instance_id(params.instance.id()) - .fetch() + let (.., authz_instance) = LookupPath::new(&opctx, &datastore) + .instance_id(instance_id) + .lookup_for(authz::Action::Modify) .await .map_err(ActionError::action_failed)?; @@ -485,7 +551,8 @@ async fn sis_ensure_running( &opctx, &authz_instance, &db_instance, - InstanceStateRequested::Running, + &Some(db_vmm), + crate::app::instance::InstanceStateChangeRequest::Run, ) .await .map_err(ActionError::action_failed)?; @@ -495,11 +562,8 @@ async fn sis_ensure_running( #[cfg(test)] mod test { + use crate::app::{saga::create_saga_dag, sagas::test_helpers}; use crate::external_api::params; - use crate::{ - app::{saga::create_saga_dag, sagas::test_helpers}, - Nexus, TestInterfaces as _, - }; use dropshot::test_util::ClientTestContext; use nexus_db_queries::authn; use nexus_test_utils::resource_helpers::{ @@ -509,8 +573,6 @@ mod test { use omicron_common::api::external::{ ByteCount, IdentityMetadataCreateParams, InstanceCpuCount, }; - use sled_agent_client::TestInterfaces as _; - use std::sync::Arc; use uuid::Uuid; use super::*; @@ -553,35 +615,6 @@ mod test { .await } - async fn fetch_db_instance( - cptestctx: &ControlPlaneTestContext, - opctx: &nexus_db_queries::context::OpContext, - id: Uuid, - ) -> nexus_db_model::Instance { - let datastore = cptestctx.server.apictx().nexus.datastore().clone(); - let (.., db_instance) = LookupPath::new(&opctx, &datastore) - .instance_id(id) - .fetch() - .await - .expect("test instance should be present in datastore"); - - info!(&cptestctx.logctx.log, "refetched instance from db"; - "instance" => ?db_instance); - - db_instance - } - - async fn instance_simulate( - cptestctx: &ControlPlaneTestContext, - nexus: &Arc, - instance_id: &Uuid, - ) { - info!(&cptestctx.logctx.log, "Poking simulated instance"; - "instance_id" => %instance_id); - let sa = nexus.instance_sled_by_id(instance_id).await.unwrap(); - sa.instance_finish_transition(*instance_id).await; - } - #[nexus_test(server = crate::Server)] async fn test_saga_basic_usage_succeeds( cptestctx: &ControlPlaneTestContext, @@ -592,22 +625,32 @@ mod test { let opctx = test_helpers::test_opctx(cptestctx); let instance = create_instance(client).await; let db_instance = - fetch_db_instance(cptestctx, &opctx, instance.identity.id).await; + test_helpers::instance_fetch(cptestctx, instance.identity.id) + .await + .instance() + .clone(); let params = Params { serialized_authn: authn::saga::Serialized::for_opctx(&opctx), - instance: db_instance, - ensure_network: true, + db_instance, }; let dag = create_saga_dag::(params).unwrap(); let saga = nexus.create_runnable_saga(dag).await.unwrap(); nexus.run_saga(saga).await.expect("Start saga should succeed"); - instance_simulate(cptestctx, nexus, &instance.identity.id).await; - let db_instance = - fetch_db_instance(cptestctx, &opctx, instance.identity.id).await; - assert_eq!(db_instance.runtime().state.0, InstanceState::Running); + test_helpers::instance_simulate(cptestctx, &instance.identity.id).await; + let vmm_state = + test_helpers::instance_fetch(cptestctx, instance.identity.id) + .await + .vmm() + .as_ref() + .expect("running instance should have a vmm") + .runtime + .state + .0; + + assert_eq!(vmm_state, InstanceState::Running); } #[nexus_test(server = crate::Server)] @@ -630,18 +673,16 @@ mod test { || { Box::pin({ async { - let db_instance = fetch_db_instance( + let db_instance = test_helpers::instance_fetch( cptestctx, - &opctx, instance.identity.id, ) - .await; + .await.instance().clone(); Params { serialized_authn: authn::saga::Serialized::for_opctx(&opctx), - instance: db_instance, - ensure_network: true, + db_instance, } } }) @@ -649,20 +690,20 @@ mod test { || { Box::pin({ async { - let new_db_instance = fetch_db_instance( + let new_db_instance = test_helpers::instance_fetch( cptestctx, - &opctx, instance.identity.id, ) - .await; + .await.instance().clone(); info!(log, "fetched instance runtime state after saga execution"; "instance_id" => %instance.identity.id, "instance_runtime" => ?new_db_instance.runtime()); + assert!(new_db_instance.runtime().propolis_id.is_none()); assert_eq!( - new_db_instance.runtime().state.0, + new_db_instance.runtime().nexus_state.0, InstanceState::Stopped ); } @@ -682,20 +723,29 @@ mod test { let opctx = test_helpers::test_opctx(cptestctx); let instance = create_instance(client).await; let db_instance = - fetch_db_instance(cptestctx, &opctx, instance.identity.id).await; + test_helpers::instance_fetch(cptestctx, instance.identity.id) + .await + .instance() + .clone(); let params = Params { serialized_authn: authn::saga::Serialized::for_opctx(&opctx), - instance: db_instance, - ensure_network: true, + db_instance, }; let dag = create_saga_dag::(params).unwrap(); test_helpers::actions_succeed_idempotently(nexus, dag).await; - instance_simulate(cptestctx, nexus, &instance.identity.id).await; - let new_db_instance = - fetch_db_instance(cptestctx, &opctx, instance.identity.id).await; - - assert_eq!(new_db_instance.runtime().state.0, InstanceState::Running); + test_helpers::instance_simulate(cptestctx, &instance.identity.id).await; + let vmm_state = + test_helpers::instance_fetch(cptestctx, instance.identity.id) + .await + .vmm() + .as_ref() + .expect("running instance should have a vmm") + .runtime + .state + .0; + + assert_eq!(vmm_state, InstanceState::Running); } } diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs index 8a9fc69f0e..88778e3573 100644 --- a/nexus/src/app/sagas/mod.rs +++ b/nexus/src/app/sagas/mod.rs @@ -23,6 +23,7 @@ pub mod disk_create; pub mod disk_delete; pub mod finalize_disk; pub mod import_blocks_from_url; +mod instance_common; pub mod instance_create; pub mod instance_delete; pub mod instance_migrate; @@ -369,7 +370,7 @@ where )) } - // Anything elses is a permanent error + // Anything else is a permanent error _ => Err(backoff::BackoffError::Permanent( progenitor_client::Error::ErrorResponse( response_value, diff --git a/nexus/src/app/sagas/snapshot_create.rs b/nexus/src/app/sagas/snapshot_create.rs index 0b3c5c99d7..5a686b2f3d 100644 --- a/nexus/src/app/sagas/snapshot_create.rs +++ b/nexus/src/app/sagas/snapshot_create.rs @@ -130,7 +130,7 @@ pub(crate) struct Params { pub silo_id: Uuid, pub project_id: Uuid, pub disk_id: Uuid, - pub use_the_pantry: bool, + pub attached_instance_and_sled: Option<(Uuid, Uuid)>, pub create_params: params::SnapshotCreate, } @@ -251,7 +251,8 @@ impl NexusSaga for SagaSnapshotCreate { // (DB) Tracks virtual resource provisioning. builder.append(space_account_action()); - if !params.use_the_pantry { + let use_the_pantry = params.attached_instance_and_sled.is_none(); + if !use_the_pantry { // (Sleds) If the disk is attached to an instance, send a // snapshot request to sled-agent to create a ZFS snapshot. builder.append(send_snapshot_request_to_sled_agent_action()); @@ -283,7 +284,7 @@ impl NexusSaga for SagaSnapshotCreate { // (DB) Mark snapshot as "ready" builder.append(finalize_snapshot_record_action()); - if params.use_the_pantry { + if use_the_pantry { // (Pantry) Set the state back to Detached // // This has to be the last saga node! Otherwise, concurrent @@ -669,67 +670,47 @@ async fn ssc_send_snapshot_request_to_sled_agent( let log = sagactx.user_data().log(); let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let snapshot_id = sagactx.lookup::("snapshot_id")?; - // Find if this disk is attached to an instance - let (.., disk) = LookupPath::new(&opctx, &osagactx.datastore()) - .disk_id(params.disk_id) - .fetch() - .await - .map_err(ActionError::action_failed)?; - - match disk.runtime().attach_instance_id { - Some(instance_id) => { - info!(log, "disk {} instance is {}", disk.id(), instance_id); - - // Get the instance's sled agent client - let (.., instance) = LookupPath::new(&opctx, &osagactx.datastore()) - .instance_id(instance_id) - .fetch() - .await - .map_err(ActionError::action_failed)?; + // If this node was reached, the saga initiator thought the disk was + // attached to an instance that was running on a specific sled. Contact that + // sled and ask it to initiate a snapshot. Note that this is best-effort: + // the instance may have stopped (or may be have stopped, had the disk + // detached, and resumed running on the same sled) while the saga was + // executing. + let (instance_id, sled_id) = + params.attached_instance_and_sled.ok_or_else(|| { + ActionError::action_failed(Error::internal_error( + "snapshot saga in send_snapshot_request_to_sled_agent but no \ + instance/sled pair was provided", + )) + })?; - let sled_agent_client = osagactx - .nexus() - .instance_sled(&instance) - .await - .map_err(ActionError::action_failed)?; + info!(log, "asking for disk snapshot from Propolis via sled agent"; + "disk_id" => %params.disk_id, + "instance_id" => %instance_id, + "sled_id" => %sled_id); - info!(log, "instance {} sled agent created ok", instance_id); + let sled_agent_client = osagactx + .nexus() + .sled_client(&sled_id) + .await + .map_err(ActionError::action_failed)?; - // Send a snapshot request to propolis through sled agent - retry_until_known_result(log, || async { - sled_agent_client - .instance_issue_disk_snapshot_request( - &instance.id(), - &disk.id(), - &InstanceIssueDiskSnapshotRequestBody { snapshot_id }, - ) - .await - }) + retry_until_known_result(log, || async { + sled_agent_client + .instance_issue_disk_snapshot_request( + &instance_id, + ¶ms.disk_id, + &InstanceIssueDiskSnapshotRequestBody { snapshot_id }, + ) .await - .map_err(|e| e.to_string()) - .map_err(ActionError::action_failed)?; - Ok(()) - } - - None => { - // This branch shouldn't be seen unless there's a detach that occurs - // after the saga starts. - error!(log, "disk {} not attached to an instance!", disk.id()); + }) + .await + .map_err(|e| e.to_string()) + .map_err(ActionError::action_failed)?; - Err(ActionError::action_failed(Error::ServiceUnavailable { - internal_message: - "disk detached after snapshot_create saga started!" - .to_string(), - })) - } - } + Ok(()) } async fn ssc_send_snapshot_request_to_sled_agent_undo( @@ -1566,7 +1547,6 @@ mod test { use crate::app::saga::create_saga_dag; use crate::app::sagas::test_helpers; - use crate::app::test_interfaces::TestInterfaces; use crate::external_api::shared::IpRange; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::{ @@ -1574,6 +1554,7 @@ mod test { }; use dropshot::test_util::ClientTestContext; use nexus_db_queries::context::OpContext; + use nexus_db_queries::db::datastore::InstanceAndActiveVmm; use nexus_db_queries::db::DataStore; use nexus_test_utils::resource_helpers::create_disk; use nexus_test_utils::resource_helpers::create_ip_pool; @@ -1810,14 +1791,14 @@ mod test { project_id: Uuid, disk_id: Uuid, disk: NameOrId, - use_the_pantry: bool, + instance_and_sled: Option<(Uuid, Uuid)>, ) -> Params { Params { serialized_authn: authn::saga::Serialized::for_opctx(opctx), silo_id, project_id, disk_id, - use_the_pantry, + attached_instance_and_sled: instance_and_sled, create_params: params::SnapshotCreate { identity: IdentityMetadataCreateParams { name: "my-snapshot".parse().expect("Invalid disk name"), @@ -1866,7 +1847,7 @@ mod test { project_id, disk_id, Name::from_str(DISK_NAME).unwrap().into(), - true, + None, ); let dag = create_saga_dag::(params).unwrap(); let runnable_saga = nexus.create_runnable_saga(dag).await.unwrap(); @@ -1941,7 +1922,7 @@ mod test { cptestctx: &ControlPlaneTestContext, client: &ClientTestContext, disks_to_attach: Vec, - ) { + ) -> InstanceAndActiveVmm { let instances_url = format!("/v1/instances?project={}", PROJECT_NAME,); let instance: Instance = object_create( client, @@ -1966,11 +1947,49 @@ mod test { ) .await; - // cannot snapshot attached disk for instance in state starting + // Read out the instance's assigned sled, then poke the instance to get + // it from the Starting state to the Running state so the test disk can + // be snapshotted. let nexus = &cptestctx.server.apictx().nexus; - let sa = - nexus.instance_sled_by_id(&instance.identity.id).await.unwrap(); + let opctx = test_opctx(&cptestctx); + let (.., authz_instance) = LookupPath::new(&opctx, nexus.datastore()) + .instance_id(instance.identity.id) + .lookup_for(authz::Action::Read) + .await + .unwrap(); + + let instance_state = nexus + .datastore() + .instance_fetch_with_vmm(&opctx, &authz_instance) + .await + .unwrap(); + + let sled_id = instance_state + .sled_id() + .expect("starting instance should have a sled"); + let sa = nexus.sled_client(&sled_id).await.unwrap(); + sa.instance_finish_transition(instance.identity.id).await; + let instance_state = nexus + .datastore() + .instance_fetch_with_vmm(&opctx, &authz_instance) + .await + .unwrap(); + + let new_state = instance_state + .vmm() + .as_ref() + .expect("running instance should have a sled") + .runtime + .state + .0; + + assert_eq!( + new_state, + omicron_common::api::external::InstanceState::Running + ); + + instance_state } #[nexus_test(server = crate::Server)] @@ -2053,8 +2072,8 @@ mod test { // since this is just a test, bypass the normal // attachment machinery and just update the disk's // database record directly. - if !use_the_pantry { - setup_test_instance( + let instance_and_sled = if !use_the_pantry { + let state = setup_test_instance( cptestctx, client, vec![params::InstanceDiskAttachment::Attach( @@ -2065,7 +2084,15 @@ mod test { )], ) .await; - } + + let sled_id = state + .sled_id() + .expect("running instance should have a vmm"); + + Some((state.instance().id(), sled_id)) + } else { + None + }; new_test_params( &opctx, @@ -2073,7 +2100,7 @@ mod test { project_id, disk_id, Name::from_str(DISK_NAME).unwrap().into(), - use_the_pantry, + instance_and_sled, ) } }) @@ -2169,8 +2196,8 @@ mod test { project_id, disk_id, Name::from_str(DISK_NAME).unwrap().into(), - // set use_the_pantry to true, disk is unattached at time of saga creation - true, + // The disk isn't attached at this time, so don't supply a sled. + None, ); let dag = create_saga_dag::(params).unwrap(); @@ -2233,8 +2260,8 @@ mod test { project_id, disk_id, Name::from_str(DISK_NAME).unwrap().into(), - // set use_the_pantry to true, disk is unattached at time of saga creation - true, + // The disk isn't attached at this time, so don't supply a sled. + None, ); let dag = create_saga_dag::(params).unwrap(); @@ -2272,14 +2299,23 @@ mod test { let silo_id = authz_silo.id(); let project_id = authz_project.id(); + // Synthesize an instance ID to pass to the saga, but use the default + // test sled ID. This will direct a snapshot request to the simulated + // sled agent specifying an instance it knows nothing about, which is + // equivalent to creating an instance, attaching the test disk, creating + // the saga, stopping the instance, detaching the disk, and then letting + // the saga run. + let fake_instance_id = Uuid::new_v4(); + let fake_sled_id = + Uuid::parse_str(nexus_test_utils::SLED_AGENT_UUID).unwrap(); + let params = new_test_params( &opctx, silo_id, project_id, disk_id, Name::from_str(DISK_NAME).unwrap().into(), - // set use_the_pantry to true, disk is attached at time of saga creation - false, + Some((fake_instance_id, fake_sled_id)), ); let dag = create_saga_dag::(params).unwrap(); @@ -2303,19 +2339,10 @@ mod test { .await .expect("failed to detach disk")); - // Actually run the saga + // Actually run the saga. This should fail. let output = nexus.run_saga(runnable_saga).await; - // Expect to see 503 - match output { - Err(e) => { - assert!(matches!(e, Error::ServiceUnavailable { .. })); - } - - Ok(_) => { - assert!(false); - } - } + assert!(output.is_err()); // Attach the disk to an instance, then rerun the saga populate_ip_pool( @@ -2331,7 +2358,7 @@ mod test { ) .await; - setup_test_instance( + let instance_state = setup_test_instance( cptestctx, client, vec![params::InstanceDiskAttachment::Attach( @@ -2342,6 +2369,10 @@ mod test { ) .await; + let sled_id = instance_state + .sled_id() + .expect("running instance should have a vmm"); + // Rerun the saga let params = new_test_params( &opctx, @@ -2349,8 +2380,7 @@ mod test { project_id, disk_id, Name::from_str(DISK_NAME).unwrap().into(), - // set use_the_pantry to false, disk is attached at time of saga creation - false, + Some((instance_state.instance().id(), sled_id)), ); let dag = create_saga_dag::(params).unwrap(); diff --git a/nexus/src/app/sagas/test_helpers.rs b/nexus/src/app/sagas/test_helpers.rs index aa9334b682..eccb013b66 100644 --- a/nexus/src/app/sagas/test_helpers.rs +++ b/nexus/src/app/sagas/test_helpers.rs @@ -15,7 +15,11 @@ use async_bb8_diesel::{ }; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use futures::future::BoxFuture; -use nexus_db_queries::{context::OpContext, db::DataStore}; +use nexus_db_queries::{ + authz, + context::OpContext, + db::{datastore::InstanceAndActiveVmm, lookup::LookupPath, DataStore}, +}; use nexus_types::identity::Resource; use omicron_common::api::external::NameOrId; use sled_agent_client::TestInterfaces as _; @@ -123,7 +127,12 @@ pub(crate) async fn instance_simulate( info!(&cptestctx.logctx.log, "Poking simulated instance"; "instance_id" => %instance_id); let nexus = &cptestctx.server.apictx().nexus; - let sa = nexus.instance_sled_by_id(instance_id).await.unwrap(); + let sa = nexus + .instance_sled_by_id(instance_id) + .await + .unwrap() + .expect("instance must be on a sled to simulate a state change"); + sa.instance_finish_transition(*instance_id).await; } @@ -147,10 +156,38 @@ pub(crate) async fn instance_simulate_by_name( let instance_lookup = nexus.instance_lookup(&opctx, instance_selector).unwrap(); let (.., instance) = instance_lookup.fetch().await.unwrap(); - let sa = nexus.instance_sled_by_id(&instance.id()).await.unwrap(); + let sa = nexus + .instance_sled_by_id(&instance.id()) + .await + .unwrap() + .expect("instance must be on a sled to simulate a state change"); sa.instance_finish_transition(instance.id()).await; } +pub async fn instance_fetch( + cptestctx: &ControlPlaneTestContext, + instance_id: Uuid, +) -> InstanceAndActiveVmm { + let datastore = cptestctx.server.apictx().nexus.datastore().clone(); + let opctx = test_opctx(&cptestctx); + let (.., authz_instance) = LookupPath::new(&opctx, &datastore) + .instance_id(instance_id) + .lookup_for(authz::Action::Read) + .await + .expect("test instance should be present in datastore"); + + let db_state = datastore + .instance_fetch_with_vmm(&opctx, &authz_instance) + .await + .expect("test instance's info should be fetchable"); + + info!(&cptestctx.logctx.log, "refetched instance info from db"; + "instance_id" => %instance_id, + "instance_and_vmm" => ?db_state); + + db_state +} + /// Tests that the saga described by `dag` succeeds if each of its nodes is /// repeated. /// diff --git a/nexus/src/app/snapshot.rs b/nexus/src/app/snapshot.rs index 06ac140606..0c90ac31fb 100644 --- a/nexus/src/app/snapshot.rs +++ b/nexus/src/app/snapshot.rs @@ -93,41 +93,43 @@ impl super::Nexus { // If there isn't a running propolis, Nexus needs to use the Crucible // Pantry to make this snapshot - let use_the_pantry = if let Some(attach_instance_id) = + let instance_and_sled = if let Some(attach_instance_id) = &db_disk.runtime_state.attach_instance_id { - let (.., db_instance) = LookupPath::new(opctx, &self.db_datastore) - .instance_id(*attach_instance_id) - .fetch_for(authz::Action::Read) + let (.., authz_instance) = + LookupPath::new(opctx, &self.db_datastore) + .instance_id(*attach_instance_id) + .lookup_for(authz::Action::Read) + .await?; + + let instance_state = self + .datastore() + .instance_fetch_with_vmm(&opctx, &authz_instance) .await?; - let instance_state: InstanceState = db_instance.runtime().state.0; - - match instance_state { - // If there's a propolis running, use that - InstanceState::Running | - // Rebooting doesn't deactivate the volume - InstanceState::Rebooting - => false, - - // If there's definitely no propolis running, then use the - // pantry - InstanceState::Stopped | InstanceState::Destroyed => true, - - // If there *may* be a propolis running, then fail: we can't - // know if that propolis has activated the Volume or not, or if - // it's in the process of deactivating. - _ => { - return Err( - Error::invalid_request( - &format!("cannot snapshot attached disk for instance in state {}", instance_state) - ) - ); - } + match instance_state.vmm().as_ref() { + None => None, + Some(vmm) => match vmm.runtime.state.0 { + // If the VM might be running, or it's rebooting (which + // doesn't deactivate the volume), send the snapshot request + // to the relevant VMM. Otherwise, there's no way to know if + // the instance has attached the volume or is in the process + // of detaching it, so bail. + InstanceState::Running | InstanceState::Rebooting => { + Some((*attach_instance_id, vmm.sled_id)) + } + _ => { + return Err(Error::invalid_request(&format!( + "cannot snapshot attached disk for instance in \ + state {}", + vmm.runtime.state.0 + ))); + } + }, } } else { // This disk is not attached to an instance, use the pantry. - true + None }; let saga_params = sagas::snapshot_create::Params { @@ -135,7 +137,7 @@ impl super::Nexus { silo_id: authz_silo.id(), project_id: authz_project.id(), disk_id: authz_disk.id(), - use_the_pantry, + attached_instance_and_sled: instance_and_sled, create_params: params.clone(), }; diff --git a/nexus/src/app/test_interfaces.rs b/nexus/src/app/test_interfaces.rs index 17ea205cbb..486569333e 100644 --- a/nexus/src/app/test_interfaces.rs +++ b/nexus/src/app/test_interfaces.rs @@ -22,17 +22,20 @@ pub trait TestInterfaces { async fn instance_sled_by_id( &self, id: &Uuid, - ) -> Result, Error>; + ) -> Result>, Error>; - /// Returns the SledAgentClient for a Disk from its id. + /// Returns the SledAgentClient for the sled running an instance to which a + /// disk is attached. async fn disk_sled_by_id( &self, id: &Uuid, - ) -> Result, Error>; + ) -> Result>, Error>; /// Returns the supplied instance's current active sled ID. - async fn instance_sled_id(&self, instance_id: &Uuid) - -> Result; + async fn instance_sled_id( + &self, + instance_id: &Uuid, + ) -> Result, Error>; async fn set_disk_as_faulted(&self, disk_id: &Uuid) -> Result; @@ -48,22 +51,19 @@ impl TestInterfaces for super::Nexus { async fn instance_sled_by_id( &self, id: &Uuid, - ) -> Result, Error> { - let opctx = OpContext::for_tests( - self.log.new(o!()), - Arc::clone(&self.db_datastore), - ); - let (.., db_instance) = LookupPath::new(&opctx, &self.db_datastore) - .instance_id(*id) - .fetch() - .await?; - self.instance_sled(&db_instance).await + ) -> Result>, Error> { + let sled_id = self.instance_sled_id(id).await?; + if let Some(sled_id) = sled_id { + Ok(Some(self.sled_client(&sled_id).await?)) + } else { + Ok(None) + } } async fn disk_sled_by_id( &self, id: &Uuid, - ) -> Result, Error> { + ) -> Result>, Error> { let opctx = OpContext::for_tests( self.log.new(o!()), Arc::clone(&self.db_datastore), @@ -72,23 +72,27 @@ impl TestInterfaces for super::Nexus { .disk_id(*id) .fetch() .await?; - let (.., db_instance) = LookupPath::new(&opctx, &self.db_datastore) - .instance_id(db_disk.runtime().attach_instance_id.unwrap()) - .fetch() - .await?; - self.instance_sled(&db_instance).await + + self.instance_sled_by_id(&db_disk.runtime().attach_instance_id.unwrap()) + .await } - async fn instance_sled_id(&self, id: &Uuid) -> Result { + async fn instance_sled_id(&self, id: &Uuid) -> Result, Error> { let opctx = OpContext::for_tests( self.log.new(o!()), Arc::clone(&self.db_datastore), ); - let (.., db_instance) = LookupPath::new(&opctx, &self.db_datastore) + + let (.., authz_instance) = LookupPath::new(&opctx, &self.db_datastore) .instance_id(*id) - .fetch() + .lookup_for(nexus_db_queries::authz::Action::Read) .await?; - Ok(db_instance.runtime().sled_id) + + Ok(self + .datastore() + .instance_fetch_with_vmm(&opctx, &authz_instance) + .await? + .sled_id()) } async fn set_disk_as_faulted(&self, disk_id: &Uuid) -> Result { diff --git a/nexus/src/cidata.rs b/nexus/src/cidata.rs index d35b3f8256..8f776501b6 100644 --- a/nexus/src/cidata.rs +++ b/nexus/src/cidata.rs @@ -21,7 +21,7 @@ impl InstanceCiData for Instance { // cloud-init meta-data is YAML, but YAML is a strict superset of JSON. let meta_data = serde_json::to_vec(&MetaData { instance_id: self.id(), - local_hostname: &self.runtime().hostname, + local_hostname: &self.hostname, public_keys, }) .map_err(|_| Error::internal_error("failed to serialize meta-data"))?; diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index ac5cf76775..1fddfba85b 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -1926,8 +1926,13 @@ async fn instance_view( }; let instance_lookup = nexus.instance_lookup(&opctx, instance_selector)?; - let (.., instance) = instance_lookup.fetch().await?; - Ok(HttpResponseOk(instance.into())) + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let instance_and_vmm = nexus + .datastore() + .instance_fetch_with_vmm(&opctx, &authz_instance) + .await?; + Ok(HttpResponseOk(instance_and_vmm.into())) }; apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } @@ -2110,7 +2115,7 @@ async fn instance_serial_console( let instance_lookup = nexus.instance_lookup(&opctx, instance_selector)?; let data = nexus - .instance_serial_console_data(&instance_lookup, &query) + .instance_serial_console_data(&opctx, &instance_lookup, &query) .await?; Ok(HttpResponseOk(data)) }; @@ -2148,6 +2153,7 @@ async fn instance_serial_console_stream( Ok(instance_lookup) => { nexus .instance_serial_console_stream( + &opctx, client_stream, &instance_lookup, &query, diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs index a99d386349..ebb21feb40 100644 --- a/nexus/src/internal_api/http_entrypoints.rs +++ b/nexus/src/internal_api/http_entrypoints.rs @@ -34,8 +34,8 @@ use omicron_common::api::external::http_pagination::PaginatedById; use omicron_common::api::external::http_pagination::ScanById; use omicron_common::api::external::http_pagination::ScanParams; use omicron_common::api::internal::nexus::DiskRuntimeState; -use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::api::internal::nexus::ProducerEndpoint; +use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::UpdateArtifactId; use oximeter::types::ProducerResults; use oximeter_producer::{collect, ProducerIdPathParams}; @@ -250,7 +250,7 @@ struct InstancePathParam { async fn cpapi_instances_put( rqctx: RequestContext>, path_params: Path, - new_runtime_state: TypedBody, + new_runtime_state: TypedBody, ) -> Result { let apictx = rqctx.context(); let nexus = &apictx.nexus; diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index 20f4b90b1b..71a3977192 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -172,7 +172,11 @@ async fn set_instance_state( } async fn instance_simulate(nexus: &Arc, id: &Uuid) { - let sa = nexus.instance_sled_by_id(id).await.unwrap(); + let sa = nexus + .instance_sled_by_id(id) + .await + .unwrap() + .expect("instance must be on a sled to simulate a state change"); sa.instance_finish_transition(*id).await; } diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 83fff2fbab..b8fcc9f2cb 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -87,6 +87,10 @@ fn get_instance_url(instance_name: &str) -> String { format!("/v1/instances/{}?{}", instance_name, get_project_selector()) } +fn get_instance_start_url(instance_name: &str) -> String { + format!("/v1/instances/{}/start?{}", instance_name, get_project_selector()) +} + fn get_disks_url() -> String { format!("/v1/disks?{}", get_project_selector()) } @@ -574,12 +578,20 @@ async fn test_instance_start_creates_networking_state( let opctx = OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); - let (.., authz_instance, db_instance) = LookupPath::new(&opctx, &datastore) + let (.., authz_instance) = LookupPath::new(&opctx, &datastore) .instance_id(instance.identity.id) - .fetch() + .lookup_for(nexus_db_queries::authz::Action::Read) .await .unwrap(); + let instance_state = datastore + .instance_fetch_with_vmm(&opctx, &authz_instance) + .await + .unwrap(); + + let sled_id = + instance_state.sled_id().expect("running instance should have a sled"); + let guest_nics = datastore .derive_guest_network_interface_info(&opctx, &authz_instance) .await @@ -589,7 +601,7 @@ async fn test_instance_start_creates_networking_state( for agent in &sled_agents { // TODO(#3107) Remove this bifurcation when Nexus programs all mappings // itself. - if agent.id != db_instance.runtime().sled_id { + if agent.id != sled_id { assert_sled_v2p_mappings( agent, &nics[0], @@ -645,7 +657,12 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { let instance_next = instance_get(&client, &instance_url).await; assert_eq!(instance_next.runtime.run_state, InstanceState::Running); - let original_sled = nexus.instance_sled_id(&instance_id).await.unwrap(); + let original_sled = nexus + .instance_sled_id(&instance_id) + .await + .unwrap() + .expect("running instance should have a sled"); + let dst_sled_id = if original_sled == default_sled_id { other_sled_id } else { @@ -666,7 +683,12 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { .parsed_body::() .unwrap(); - let current_sled = nexus.instance_sled_id(&instance_id).await.unwrap(); + let current_sled = nexus + .instance_sled_id(&instance_id) + .await + .unwrap() + .expect("running instance should have a sled"); + assert_eq!(current_sled, original_sled); // Explicitly simulate the migration action on the target. Simulated @@ -678,7 +700,12 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { let instance = instance_get(&client, &instance_url).await; assert_eq!(instance.runtime.run_state, InstanceState::Running); - let current_sled = nexus.instance_sled_id(&instance_id).await.unwrap(); + let current_sled = nexus + .instance_sled_id(&instance_id) + .await + .unwrap() + .expect("migrated instance should still have a sled"); + assert_eq!(current_sled, dst_sled_id); } @@ -752,7 +779,11 @@ async fn test_instance_migrate_v2p(cptestctx: &ControlPlaneTestContext) { .derive_guest_network_interface_info(&opctx, &authz_instance) .await .unwrap(); - let original_sled_id = nexus.instance_sled_id(&instance_id).await.unwrap(); + let original_sled_id = nexus + .instance_sled_id(&instance_id) + .await + .unwrap() + .expect("running instance should have a sled"); let mut sled_agents = vec![cptestctx.sled_agent.sled_agent.clone()]; sled_agents.extend(other_sleds.iter().map(|tup| tup.1.sled_agent.clone())); @@ -806,7 +837,11 @@ async fn test_instance_migrate_v2p(cptestctx: &ControlPlaneTestContext) { instance_simulate_on_sled(cptestctx, nexus, dst_sled_id, instance_id).await; let instance = instance_get(&client, &instance_url).await; assert_eq!(instance.runtime.run_state, InstanceState::Running); - let current_sled = nexus.instance_sled_id(&instance_id).await.unwrap(); + let current_sled = nexus + .instance_sled_id(&instance_id) + .await + .unwrap() + .expect("migrated instance should have a sled"); assert_eq!(current_sled, dst_sled_id); for sled_agent in &sled_agents { @@ -1050,7 +1085,7 @@ async fn test_instances_delete_fails_when_running_succeeds_when_stopped( .unwrap(); assert_eq!( error.message, - "instance cannot be deleted in state \"running\"" + "cannot delete instance: instance is running or has not yet fully stopped" ); // Stop the instance @@ -2816,16 +2851,22 @@ async fn test_disks_detached_when_instance_destroyed( assert!(matches!(disk.state, DiskState::Attached(_))); } - // Stop and delete instance + // Stash the instance's current sled agent for later disk simulation. This + // needs to be done before the instance is stopped and dissociated from its + // sled. let instance_url = format!("/v1/instances/nfs?project={}", PROJECT_NAME); - let instance = - instance_post(&client, instance_name, InstanceOp::Stop).await; - + let instance = instance_get(&client, &instance_url).await; let apictx = &cptestctx.server.apictx(); let nexus = &apictx.nexus; + let sa = nexus + .instance_sled_by_id(&instance.identity.id) + .await + .unwrap() + .expect("instance should be on a sled while it's running"); - // Store the sled agent for this instance for later disk simulation - let sa = nexus.instance_sled_by_id(&instance.identity.id).await.unwrap(); + // Stop and delete instance + let instance = + instance_post(&client, instance_name, InstanceOp::Stop).await; instance_simulate(nexus, &instance.identity.id).await; let instance = instance_get(&client, &instance_url).await; @@ -3042,20 +3083,40 @@ async fn test_instances_memory_greater_than_max_size( assert!(error.message.contains("memory must be less than")); } -async fn expect_instance_creation_fail_unavailable( +async fn expect_instance_start_fail_unavailable( client: &ClientTestContext, - url_instances: &str, - instance_params: ¶ms::InstanceCreate, + instance_name: &str, ) { - let builder = - RequestBuilder::new(client, http::Method::POST, &url_instances) - .body(Some(&instance_params)) - .expect_status(Some(http::StatusCode::SERVICE_UNAVAILABLE)); + let builder = RequestBuilder::new( + client, + http::Method::POST, + &get_instance_start_url(instance_name), + ) + .expect_status(Some(http::StatusCode::SERVICE_UNAVAILABLE)); + NexusRequest::new(builder) .authn_as(AuthnMode::PrivilegedUser) .execute() .await - .expect("Expected instance creation to fail with SERVICE_UNAVAILABLE!"); + .expect("Expected instance start to fail with SERVICE_UNAVAILABLE"); +} + +async fn expect_instance_start_ok( + client: &ClientTestContext, + instance_name: &str, +) { + let builder = RequestBuilder::new( + client, + http::Method::POST, + &get_instance_start_url(instance_name), + ) + .expect_status(Some(http::StatusCode::ACCEPTED)); + + NexusRequest::new(builder) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("Expected instance start to succeed with 202 Accepted"); } async fn expect_instance_creation_ok( @@ -3074,17 +3135,6 @@ async fn expect_instance_creation_ok( .expect("Expected instance creation to work!"); } -async fn expect_instance_deletion_ok( - client: &ClientTestContext, - url_instances: &str, -) { - NexusRequest::object_delete(client, &url_instances) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .unwrap(); -} - #[nexus_test] async fn test_cannot_provision_instance_beyond_cpu_capacity( cptestctx: &ControlPlaneTestContext, @@ -3093,59 +3143,65 @@ async fn test_cannot_provision_instance_beyond_cpu_capacity( create_project(client, PROJECT_NAME).await; populate_ip_pool(&client, "default", None).await; - let too_many_cpus = InstanceCpuCount::try_from(i64::from( - nexus_test_utils::TEST_HARDWARE_THREADS + 1, - )) - .unwrap(); - let enough_cpus = InstanceCpuCount::try_from(i64::from( - nexus_test_utils::TEST_HARDWARE_THREADS, - )) - .unwrap(); + // The third item in each tuple specifies whether instance start should + // succeed or fail if all these configs are visited in order and started in + // sequence. Note that for this reason the order of these elements matters. + let configs = vec![ + ("too-many-cpus", nexus_test_utils::TEST_HARDWARE_THREADS + 1, Err(())), + ("just-right-cpus", nexus_test_utils::TEST_HARDWARE_THREADS, Ok(())), + ( + "insufficient-space", + nexus_test_utils::TEST_HARDWARE_THREADS, + Err(()), + ), + ]; - // Try to boot an instance that uses more CPUs than we have - // on our test sled setup. - let name1 = Name::try_from(String::from("test")).unwrap(); - let mut instance_params = params::InstanceCreate { - identity: IdentityMetadataCreateParams { - name: name1.clone(), - description: String::from("probably serving data"), - }, - ncpus: too_many_cpus, - memory: ByteCount::from_gibibytes_u32(4), - hostname: String::from("test"), - user_data: vec![], - network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, - external_ips: vec![], - disks: vec![], - start: false, - }; - let url_instances = get_instances_url(); + // Creating all the instances should succeed, even though there will never + // be enough space to run the too-large instance. + let mut instances = Vec::new(); + for config in &configs { + let name = Name::try_from(config.0.to_string()).unwrap(); + let ncpus = InstanceCpuCount::try_from(i64::from(config.1)).unwrap(); + let params = params::InstanceCreate { + identity: IdentityMetadataCreateParams { + name, + description: String::from("probably serving data"), + }, + ncpus, + memory: ByteCount::from_gibibytes_u32(1), + hostname: config.0.to_string(), + user_data: vec![], + network_interfaces: + params::InstanceNetworkInterfaceAttachment::Default, + external_ips: vec![], + disks: vec![], + start: false, + }; - expect_instance_creation_fail_unavailable( - client, - &url_instances, - &instance_params, - ) - .await; + let url_instances = get_instances_url(); + expect_instance_creation_ok(client, &url_instances, ¶ms).await; - // If we ask for fewer CPUs, the request should work - instance_params.ncpus = enough_cpus; - expect_instance_creation_ok(client, &url_instances, &instance_params).await; + let instance = instance_get(&client, &get_instance_url(config.0)).await; + instances.push(instance); + } - // Requesting another instance won't have enough space - let name2 = Name::try_from(String::from("test2")).unwrap(); - instance_params.identity.name = name2; - expect_instance_creation_fail_unavailable( - client, - &url_instances, - &instance_params, - ) - .await; + // Only the first properly-sized instance should be able to start. + for config in &configs { + match config.2 { + Ok(_) => expect_instance_start_ok(client, config.0).await, + Err(_) => { + expect_instance_start_fail_unavailable(client, config.0).await + } + } + } - // But if we delete the first instace, we'll have space - let url_instance = get_instance_url(&name1.to_string()); - expect_instance_deletion_ok(client, &url_instance).await; - expect_instance_creation_ok(client, &url_instances, &instance_params).await; + // Make the started instance transition to Running, shut it down, and verify + // that the other reasonably-sized instance can now start. + let nexus = &cptestctx.server.apictx().nexus; + instance_simulate(nexus, &instances[1].identity.id).await; + instances[1] = instance_post(client, configs[1].0, InstanceOp::Stop).await; + instance_simulate(nexus, &instances[1].identity.id).await; + expect_instance_start_ok(client, configs[2].0).await; } #[nexus_test] @@ -3198,57 +3254,62 @@ async fn test_cannot_provision_instance_beyond_ram_capacity( create_project(client, PROJECT_NAME).await; populate_ip_pool(&client, "default", None).await; - let too_much_ram = ByteCount::try_from( - nexus_test_utils::TEST_PHYSICAL_RAM - + u64::from(MIN_MEMORY_BYTES_PER_INSTANCE), - ) - .unwrap(); - let enough_ram = - ByteCount::try_from(nexus_test_utils::TEST_PHYSICAL_RAM).unwrap(); + let configs = vec![ + ( + "too-much-memory", + nexus_test_utils::TEST_RESERVOIR_RAM + + u64::from(MIN_MEMORY_BYTES_PER_INSTANCE), + Err(()), + ), + ("just-right-memory", nexus_test_utils::TEST_RESERVOIR_RAM, Ok(())), + ("insufficient-space", nexus_test_utils::TEST_RESERVOIR_RAM, Err(())), + ]; - // Try to boot an instance that uses more RAM than we have - // on our test sled setup. - let name1 = Name::try_from(String::from("test")).unwrap(); - let mut instance_params = params::InstanceCreate { - identity: IdentityMetadataCreateParams { - name: name1.clone(), - description: String::from("probably serving data"), - }, - ncpus: InstanceCpuCount::try_from(2).unwrap(), - memory: too_much_ram, - hostname: String::from("test"), - user_data: vec![], - network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, - external_ips: vec![], - disks: vec![], - start: false, - }; - let url_instances = get_instances_url(); - expect_instance_creation_fail_unavailable( - client, - &url_instances, - &instance_params, - ) - .await; + // Creating all the instances should succeed, even though there will never + // be enough space to run the too-large instance. + let mut instances = Vec::new(); + for config in &configs { + let name = Name::try_from(config.0.to_string()).unwrap(); + let params = params::InstanceCreate { + identity: IdentityMetadataCreateParams { + name, + description: String::from("probably serving data"), + }, + ncpus: InstanceCpuCount::try_from(i64::from(1)).unwrap(), + memory: ByteCount::try_from(config.1).unwrap(), + hostname: config.0.to_string(), + user_data: vec![], + network_interfaces: + params::InstanceNetworkInterfaceAttachment::Default, + external_ips: vec![], + disks: vec![], + start: false, + }; - // If we ask for less RAM, the request should work - instance_params.memory = enough_ram; - expect_instance_creation_ok(client, &url_instances, &instance_params).await; + let url_instances = get_instances_url(); + expect_instance_creation_ok(client, &url_instances, ¶ms).await; - // Requesting another instance won't have enough space - let name2 = Name::try_from(String::from("test2")).unwrap(); - instance_params.identity.name = name2; - expect_instance_creation_fail_unavailable( - client, - &url_instances, - &instance_params, - ) - .await; + let instance = instance_get(&client, &get_instance_url(config.0)).await; + instances.push(instance); + } - // But if we delete the first instace, we'll have space - let url_instance = get_instance_url(&name1.to_string()); - expect_instance_deletion_ok(client, &url_instance).await; - expect_instance_creation_ok(client, &url_instances, &instance_params).await; + // Only the first properly-sized instance should be able to start. + for config in &configs { + match config.2 { + Ok(_) => expect_instance_start_ok(client, config.0).await, + Err(_) => { + expect_instance_start_fail_unavailable(client, config.0).await + } + } + } + + // Make the started instance transition to Running, shut it down, and verify + // that the other reasonably-sized instance can now start. + let nexus = &cptestctx.server.apictx().nexus; + instance_simulate(nexus, &instances[1].identity.id).await; + instances[1] = instance_post(client, configs[1].0, InstanceOp::Stop).await; + instance_simulate(nexus, &instances[1].identity.id).await; + expect_instance_start_ok(client, configs[2].0).await; } #[nexus_test] @@ -3288,17 +3349,8 @@ async fn test_instance_serial(cptestctx: &ControlPlaneTestContext) { format!("not found: instance with name \"{}\"", instance_name).as_str() ); - // Create an instance. + // Create an instance and poke it to ensure it's running. let instance = create_instance(client, PROJECT_NAME, instance_name).await; - - // Now, simulate completion of instance boot and check the state reported. - // NOTE: prior to this instance_simulate call, nexus's stored propolis addr - // is one it allocated in a 'real' sled-agent IP range as part of its usual - // instance-creation saga. after we poke the new run state for the instance - // here, sled-agent-sim will send an entire updated InstanceRuntimeState - // back to nexus, including the localhost address on which the mock - // propolis-server is running, which overwrites this -- after which nexus's - // serial-console related API calls will start working. instance_simulate(nexus, &instance.identity.id).await; let instance_next = instance_get(&client, &instance_url).await; identity_eq(&instance.identity, &instance_next.identity); @@ -3308,6 +3360,29 @@ async fn test_instance_serial(cptestctx: &ControlPlaneTestContext) { > instance.runtime.time_run_state_updated ); + // Starting a simulated instance with a mock Propolis server starts the + // mock, but it serves on localhost instead of the address that was chosen + // by the instance start process. Forcibly update the VMM record to point to + // the correct IP. + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + let (.., db_instance) = LookupPath::new(&opctx, &datastore) + .instance_id(instance.identity.id) + .fetch() + .await + .unwrap(); + let propolis_id = db_instance + .runtime() + .propolis_id + .expect("running instance should have vmm"); + let localhost = std::net::IpAddr::V6(std::net::Ipv6Addr::LOCALHOST); + let updated_vmm = datastore + .vmm_overwrite_ip_for_test(&opctx, &propolis_id, localhost.into()) + .await + .unwrap(); + assert_eq!(updated_vmm.propolis_ip.ip(), localhost); + // Query serial output history endpoint // This is the first line of output generated by the mock propolis-server. let expected = "This is simulated serial console output for ".as_bytes(); @@ -3615,16 +3690,25 @@ async fn test_instance_v2p_mappings(cptestctx: &ControlPlaneTestContext) { let opctx = OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); - let (.., authz_instance, db_instance) = LookupPath::new(&opctx, &datastore) + let (.., authz_instance) = LookupPath::new(&opctx, &datastore) .instance_id(instance.identity.id) - .fetch() + .lookup_for(nexus_db_queries::authz::Action::Read) + .await + .unwrap(); + + let instance_state = datastore + .instance_fetch_with_vmm(&opctx, &authz_instance) .await .unwrap(); + let sled_id = + instance_state.sled_id().expect("running instance should have a sled"); + let guest_nics = datastore .derive_guest_network_interface_info(&opctx, &authz_instance) .await .unwrap(); + assert_eq!(guest_nics.len(), 1); let mut sled_agents: Vec<&Arc> = @@ -3634,7 +3718,7 @@ async fn test_instance_v2p_mappings(cptestctx: &ControlPlaneTestContext) { for sled_agent in &sled_agents { // TODO(#3107) Remove this bifurcation when Nexus programs all mappings // itself. - if sled_agent.id != db_instance.runtime().sled_id { + if sled_agent.id != sled_id { assert_sled_v2p_mappings( sled_agent, &nics[0], @@ -3765,7 +3849,11 @@ async fn assert_sled_v2p_mappings( /// instance, and then tell it to finish simulating whatever async transition is /// going on. pub async fn instance_simulate(nexus: &Arc, id: &Uuid) { - let sa = nexus.instance_sled_by_id(id).await.unwrap(); + let sa = nexus + .instance_sled_by_id(id) + .await + .unwrap() + .expect("instance must be on a sled to simulate a state change"); sa.instance_finish_transition(*id).await; } diff --git a/nexus/tests/integration_tests/ip_pools.rs b/nexus/tests/integration_tests/ip_pools.rs index 27f4b04290..6a633fc5e1 100644 --- a/nexus/tests/integration_tests/ip_pools.rs +++ b/nexus/tests/integration_tests/ip_pools.rs @@ -889,7 +889,11 @@ async fn test_ip_range_delete_with_allocated_external_ip_fails( .expect("Failed to stop instance"); // Simulate the transition, wait until it is in fact stopped. - let sa = nexus.instance_sled_by_id(&instance.identity.id).await.unwrap(); + let sa = nexus + .instance_sled_by_id(&instance.identity.id) + .await + .unwrap() + .expect("running instance should be on a sled"); sa.instance_finish_transition(instance.identity.id).await; // Delete the instance diff --git a/nexus/tests/integration_tests/pantry.rs b/nexus/tests/integration_tests/pantry.rs index c63f57e7fb..26e27e92ee 100644 --- a/nexus/tests/integration_tests/pantry.rs +++ b/nexus/tests/integration_tests/pantry.rs @@ -84,7 +84,11 @@ async fn set_instance_state( } async fn instance_simulate(nexus: &Arc, id: &Uuid) { - let sa = nexus.instance_sled_by_id(id).await.unwrap(); + let sa = nexus + .instance_sled_by_id(id) + .await + .unwrap() + .expect("instance must be on a sled to simulate a state change"); sa.instance_finish_transition(*id).await; } diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 1d4556e8ed..6d2595b561 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -569,7 +569,13 @@ impl InformationSchema { fn pretty_assert_eq(&self, other: &Self) { // similar_asserts gets us nice diff that only includes the relevant context. // the columns diff especially needs this: it can be 20k lines otherwise + similar_asserts::assert_eq!(self.tables, other.tables); similar_asserts::assert_eq!(self.columns, other.columns); + similar_asserts::assert_eq!(self.views, other.views); + similar_asserts::assert_eq!( + self.table_constraints, + other.table_constraints + ); similar_asserts::assert_eq!( self.check_constraints, other.check_constraints @@ -586,15 +592,9 @@ impl InformationSchema { self.referential_constraints, other.referential_constraints ); - similar_asserts::assert_eq!(self.views, other.views); similar_asserts::assert_eq!(self.statistics, other.statistics); similar_asserts::assert_eq!(self.sequences, other.sequences); similar_asserts::assert_eq!(self.pg_indexes, other.pg_indexes); - similar_asserts::assert_eq!(self.tables, other.tables); - similar_asserts::assert_eq!( - self.table_constraints, - other.table_constraints - ); } async fn new(crdb: &CockroachInstance) -> Self { diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 1ec8c1a5eb..67db222155 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -211,7 +211,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceRuntimeState" + "$ref": "#/components/schemas/SledInstanceState" } } }, @@ -3537,102 +3537,44 @@ "start_time" ] }, - "InstanceCpuCount": { - "description": "The number of CPUs in an Instance", - "type": "integer", - "format": "uint16", - "minimum": 0 - }, "InstanceRuntimeState": { - "description": "Runtime state of the Instance, including the actual running state and minimal metadata\n\nThis state is owned by the sled agent running that Instance.", + "description": "The dynamic runtime properties of an instance: its current VMM ID (if any), migration information (if any), and the instance state to report if there is no active VMM.", "type": "object", "properties": { "dst_propolis_id": { "nullable": true, - "description": "the target propolis-server during a migration of this Instance", + "description": "If a migration is active, the ID of the target VMM.", "type": "string", "format": "uuid" }, "gen": { - "description": "generation number for this state", + "description": "Generation number for this state.", "allOf": [ { "$ref": "#/components/schemas/Generation" } ] }, - "hostname": { - "description": "RFC1035-compliant hostname for the Instance.", - "type": "string" - }, - "memory": { - "description": "memory allocated for this Instance", - "allOf": [ - { - "$ref": "#/components/schemas/ByteCount" - } - ] - }, "migration_id": { "nullable": true, - "description": "migration id (if one in process)", + "description": "If a migration is active, the ID of that migration.", "type": "string", "format": "uuid" }, - "ncpus": { - "description": "number of CPUs allocated for this Instance", - "allOf": [ - { - "$ref": "#/components/schemas/InstanceCpuCount" - } - ] - }, - "propolis_addr": { - "nullable": true, - "description": "address of propolis-server running this Instance", - "type": "string" - }, - "propolis_gen": { - "description": "The generation number for the Propolis and sled identifiers for this instance.", - "allOf": [ - { - "$ref": "#/components/schemas/Generation" - } - ] - }, "propolis_id": { - "description": "which propolis-server is running this Instance", - "type": "string", - "format": "uuid" - }, - "run_state": { - "description": "runtime state of the Instance", - "allOf": [ - { - "$ref": "#/components/schemas/InstanceState" - } - ] - }, - "sled_id": { - "description": "which sled is running this Instance", + "nullable": true, + "description": "The instance's currently active VMM ID.", "type": "string", "format": "uuid" }, "time_updated": { - "description": "timestamp for this information", + "description": "Timestamp for this information.", "type": "string", "format": "date-time" } }, "required": [ "gen", - "hostname", - "memory", - "ncpus", - "propolis_gen", - "propolis_id", - "run_state", - "sled_id", "time_updated" ] }, @@ -5002,6 +4944,38 @@ "usable_physical_ram" ] }, + "SledInstanceState": { + "description": "A wrapper type containing a sled's total knowledge of the state of a specific VMM and the instance it incarnates.", + "type": "object", + "properties": { + "instance_state": { + "description": "The sled's conception of the state of the instance.", + "allOf": [ + { + "$ref": "#/components/schemas/InstanceRuntimeState" + } + ] + }, + "propolis_id": { + "description": "The ID of the VMM whose state is being reported.", + "type": "string", + "format": "uuid" + }, + "vmm_state": { + "description": "The most recent state of the sled's VMM process.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmRuntimeState" + } + ] + } + }, + "required": [ + "instance_state", + "propolis_id", + "vmm_state" + ] + }, "SledRole": { "description": "Describes the role of the sled within the rack.\n\nNote that this may change if the sled is physically moved within the rack.", "oneOf": [ @@ -5185,6 +5159,38 @@ "minLength": 1, "maxLength": 63 }, + "VmmRuntimeState": { + "description": "The dynamic runtime properties of an individual VMM process.", + "type": "object", + "properties": { + "gen": { + "description": "The generation number for this VMM's state.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "state": { + "description": "The last state reported by this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/InstanceState" + } + ] + }, + "time_updated": { + "description": "Timestamp for the VMM's state.", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "gen", + "state", + "time_updated" + ] + }, "ZpoolPutRequest": { "description": "Sent by a sled agent on startup to Nexus to request further instruction", "type": "object", diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 91f027d28c..56437ab283 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -101,7 +101,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceRuntimeState" + "$ref": "#/components/schemas/SledInstanceState" } } } @@ -231,7 +231,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceRuntimeState" + "$ref": "#/components/schemas/SledInstanceState" } } } @@ -1644,20 +1644,49 @@ "minimum": 0 }, "InstanceEnsureBody": { - "description": "The body of a request to ensure that an instance is known to a sled agent.", + "description": "The body of a request to ensure that a instance and VMM are known to a sled agent.", "type": "object", "properties": { - "initial": { + "hardware": { "description": "A description of the instance's virtual hardware and the initial runtime state this sled agent should store for this incarnation of the instance.", "allOf": [ { "$ref": "#/components/schemas/InstanceHardware" } ] + }, + "instance_runtime": { + "description": "The instance runtime state for the instance being registered.", + "allOf": [ + { + "$ref": "#/components/schemas/InstanceRuntimeState" + } + ] + }, + "propolis_addr": { + "description": "The address at which this VMM should serve a Propolis server API.", + "type": "string" + }, + "propolis_id": { + "description": "The ID of the VMM being registered. This may not be the active VMM ID in the instance runtime state (e.g. if the new VMM is going to be a migration target).", + "type": "string", + "format": "uuid" + }, + "vmm_runtime": { + "description": "The initial VMM runtime state for the VMM being registered.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmRuntimeState" + } + ] } }, "required": [ - "initial" + "hardware", + "instance_runtime", + "propolis_addr", + "propolis_id", + "vmm_runtime" ] }, "InstanceHardware": { @@ -1694,8 +1723,8 @@ "$ref": "#/components/schemas/NetworkInterface" } }, - "runtime": { - "$ref": "#/components/schemas/InstanceRuntimeState" + "properties": { + "$ref": "#/components/schemas/InstanceProperties" }, "source_nat": { "$ref": "#/components/schemas/SourceNatConfig" @@ -1706,7 +1735,7 @@ "external_ips", "firewall_rules", "nics", - "runtime", + "properties", "source_nat" ] }, @@ -1771,6 +1800,27 @@ "src_propolis_id" ] }, + "InstanceProperties": { + "description": "The \"static\" properties of an instance: information about the instance that doesn't change while the instance is running.", + "type": "object", + "properties": { + "hostname": { + "description": "RFC1035-compliant hostname for the instance.", + "type": "string" + }, + "memory": { + "$ref": "#/components/schemas/ByteCount" + }, + "ncpus": { + "$ref": "#/components/schemas/InstanceCpuCount" + } + }, + "required": [ + "hostname", + "memory", + "ncpus" + ] + }, "InstancePutMigrationIdsBody": { "description": "The body of a request to set or clear the migration identifiers from a sled agent's instance state records.", "type": "object", @@ -1785,7 +1835,7 @@ ] }, "old_runtime": { - "description": "The last runtime state known to this requestor. This request will succeed if either (a) the Propolis generation in the sled agent's runtime state matches the generation in this record, or (b) the sled agent's runtime state matches what would result from applying this request to the caller's runtime state. This latter condition provides idempotency.", + "description": "The last instance runtime state known to this requestor. This request will succeed if either (a) the state generation in the sled agent's runtime state matches the generation in this record, or (b) the sled agent's runtime state matches what would result from applying this request to the caller's runtime state. This latter condition provides idempotency.", "allOf": [ { "$ref": "#/components/schemas/InstanceRuntimeState" @@ -1823,102 +1873,50 @@ "description": "The current runtime state of the instance after handling the request to change its state. If the instance's state did not change, this field is `None`.", "allOf": [ { - "$ref": "#/components/schemas/InstanceRuntimeState" + "$ref": "#/components/schemas/SledInstanceState" } ] } } }, "InstanceRuntimeState": { - "description": "Runtime state of the Instance, including the actual running state and minimal metadata\n\nThis state is owned by the sled agent running that Instance.", + "description": "The dynamic runtime properties of an instance: its current VMM ID (if any), migration information (if any), and the instance state to report if there is no active VMM.", "type": "object", "properties": { "dst_propolis_id": { "nullable": true, - "description": "the target propolis-server during a migration of this Instance", + "description": "If a migration is active, the ID of the target VMM.", "type": "string", "format": "uuid" }, "gen": { - "description": "generation number for this state", + "description": "Generation number for this state.", "allOf": [ { "$ref": "#/components/schemas/Generation" } ] }, - "hostname": { - "description": "RFC1035-compliant hostname for the Instance.", - "type": "string" - }, - "memory": { - "description": "memory allocated for this Instance", - "allOf": [ - { - "$ref": "#/components/schemas/ByteCount" - } - ] - }, "migration_id": { "nullable": true, - "description": "migration id (if one in process)", + "description": "If a migration is active, the ID of that migration.", "type": "string", "format": "uuid" }, - "ncpus": { - "description": "number of CPUs allocated for this Instance", - "allOf": [ - { - "$ref": "#/components/schemas/InstanceCpuCount" - } - ] - }, - "propolis_addr": { - "nullable": true, - "description": "address of propolis-server running this Instance", - "type": "string" - }, - "propolis_gen": { - "description": "The generation number for the Propolis and sled identifiers for this instance.", - "allOf": [ - { - "$ref": "#/components/schemas/Generation" - } - ] - }, "propolis_id": { - "description": "which propolis-server is running this Instance", - "type": "string", - "format": "uuid" - }, - "run_state": { - "description": "runtime state of the Instance", - "allOf": [ - { - "$ref": "#/components/schemas/InstanceState" - } - ] - }, - "sled_id": { - "description": "which sled is running this Instance", + "nullable": true, + "description": "The instance's currently active VMM ID.", "type": "string", "format": "uuid" }, "time_updated": { - "description": "timestamp for this information", + "description": "Timestamp for this information.", "type": "string", "format": "date-time" } }, "required": [ "gen", - "hostname", - "memory", - "ncpus", - "propolis_gen", - "propolis_id", - "run_state", - "sled_id", "time_updated" ] }, @@ -2075,7 +2073,7 @@ "description": "The current state of the instance after handling the request to unregister it. If the instance's state did not change, this field is `None`.", "allOf": [ { - "$ref": "#/components/schemas/InstanceRuntimeState" + "$ref": "#/components/schemas/SledInstanceState" } ] } @@ -2701,6 +2699,38 @@ "vni" ] }, + "SledInstanceState": { + "description": "A wrapper type containing a sled's total knowledge of the state of a specific VMM and the instance it incarnates.", + "type": "object", + "properties": { + "instance_state": { + "description": "The sled's conception of the state of the instance.", + "allOf": [ + { + "$ref": "#/components/schemas/InstanceRuntimeState" + } + ] + }, + "propolis_id": { + "description": "The ID of the VMM whose state is being reported.", + "type": "string", + "format": "uuid" + }, + "vmm_state": { + "description": "The most recent state of the sled's VMM process.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmRuntimeState" + } + ] + } + }, + "required": [ + "instance_state", + "propolis_id", + "vmm_state" + ] + }, "SledRole": { "oneOf": [ { @@ -2834,6 +2864,38 @@ "version" ] }, + "VmmRuntimeState": { + "description": "The dynamic runtime properties of an individual VMM process.", + "type": "object", + "properties": { + "gen": { + "description": "The generation number for this VMM's state.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "state": { + "description": "The last state reported by this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/InstanceState" + } + ] + }, + "time_updated": { + "description": "Timestamp for the VMM's state.", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "gen", + "state", + "time_updated" + ] + }, "Vni": { "description": "A Geneve Virtual Network Identifier", "type": "integer", diff --git a/schema/crdb/6.0.0/README.adoc b/schema/crdb/6.0.0/README.adoc new file mode 100644 index 0000000000..e59c04fe83 --- /dev/null +++ b/schema/crdb/6.0.0/README.adoc @@ -0,0 +1,14 @@ +This upgrade turns VMM processes into first-class objects in the Omicron data +model. Instead of storing an instance's runtime state entirely in the Instance +table, Nexus stores per-VMM state and uses an Instance's active_propolis_id to +determine which VMM (if any) holds the instance's current runtime state. This +makes it much easier for Nexus to reason about the lifecycles of Propolis jobs +and their resource requirements. + +In this scheme: + +* Sled assignments and Propolis server IPs are tracked per-VMM. +* An instance may not have an active VMM at all. In that case its own `state` + column supplies the instance's logical state. +* An instance's two generation numbers (one for the reported instance state and + one for its Propolis IDs) are once again collapsed into a single number. diff --git a/schema/crdb/6.0.0/up01.sql b/schema/crdb/6.0.0/up01.sql new file mode 100644 index 0000000000..b532fc8019 --- /dev/null +++ b/schema/crdb/6.0.0/up01.sql @@ -0,0 +1,6 @@ +/* + * Drop the instance-by-sled index since there will no longer be a sled ID in + * the instance table. + */ + +DROP INDEX IF EXISTS lookup_instance_by_sled; diff --git a/schema/crdb/6.0.0/up02.sql b/schema/crdb/6.0.0/up02.sql new file mode 100644 index 0000000000..51f796f512 --- /dev/null +++ b/schema/crdb/6.0.0/up02.sql @@ -0,0 +1,13 @@ +/* + * The sled_instance view cannot be modified in place because it depends on the + * VMM table. It would be nice to define the VMM table and then alter the + * sled_instance table, but there's no way to express this correctly in the + * clean-slate DB initialization SQL (dbinit.sql) because it requires inserting + * a table into the middle of an existing sequence of table definitions. (See + * the README for more on why this causes problems.) Instead, delete the + * `sled_instance` view, then add the VMM table, then add the view back and + * leave it to `dbinit.sql` to re-create the resulting object ordering when + * creating a database from a clean slate. + */ + +DROP VIEW IF EXISTS omicron.public.sled_instance; diff --git a/schema/crdb/6.0.0/up03.sql b/schema/crdb/6.0.0/up03.sql new file mode 100644 index 0000000000..698a5f6f2d --- /dev/null +++ b/schema/crdb/6.0.0/up03.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS omicron.public.vmm ( + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + instance_id UUID NOT NULL, + state omicron.public.instance_state NOT NULL, + time_state_updated TIMESTAMPTZ NOT NULL, + state_generation INT NOT NULL, + sled_id UUID NOT NULL, + propolis_ip INET NOT NULL +); diff --git a/schema/crdb/6.0.0/up04.sql b/schema/crdb/6.0.0/up04.sql new file mode 100644 index 0000000000..b1a96ece52 --- /dev/null +++ b/schema/crdb/6.0.0/up04.sql @@ -0,0 +1,23 @@ +CREATE OR REPLACE VIEW omicron.public.sled_instance +AS SELECT + instance.id, + instance.name, + silo.name as silo_name, + project.name as project_name, + vmm.sled_id as active_sled_id, + instance.time_created, + instance.time_modified, + instance.migration_id, + instance.ncpus, + instance.memory, + vmm.state +FROM + omicron.public.instance AS instance + JOIN omicron.public.project AS project ON + instance.project_id = project.id + JOIN omicron.public.silo AS silo ON + project.silo_id = silo.id + JOIN omicron.public.vmm AS vmm ON + instance.active_propolis_id = vmm.id +WHERE + instance.time_deleted IS NULL AND vmm.time_deleted IS NULL; diff --git a/schema/crdb/6.0.0/up05.sql b/schema/crdb/6.0.0/up05.sql new file mode 100644 index 0000000000..034d2f75e8 --- /dev/null +++ b/schema/crdb/6.0.0/up05.sql @@ -0,0 +1,8 @@ +/* + * Now that the sled_instance view is up-to-date, begin to drop columns from the + * instance table that are no longer needed. This needs to be done after + * altering the sled_instance view because it's illegal to drop columns that a + * view depends on. + */ + +ALTER TABLE omicron.public.instance DROP COLUMN IF EXISTS active_sled_id; diff --git a/schema/crdb/6.0.0/up06.sql b/schema/crdb/6.0.0/up06.sql new file mode 100644 index 0000000000..42f73d82b8 --- /dev/null +++ b/schema/crdb/6.0.0/up06.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.instance DROP COLUMN IF EXISTS active_propolis_ip; diff --git a/schema/crdb/6.0.0/up07.sql b/schema/crdb/6.0.0/up07.sql new file mode 100644 index 0000000000..d8bc3cae13 --- /dev/null +++ b/schema/crdb/6.0.0/up07.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.instance DROP COLUMN IF EXISTS propolis_generation; diff --git a/schema/crdb/6.0.0/up08.sql b/schema/crdb/6.0.0/up08.sql new file mode 100644 index 0000000000..776b794a44 --- /dev/null +++ b/schema/crdb/6.0.0/up08.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.instance ALTER COLUMN active_propolis_id DROP NOT NULL; diff --git a/schema/crdb/6.0.0/up09.sql b/schema/crdb/6.0.0/up09.sql new file mode 100644 index 0000000000..1d435cec6c --- /dev/null +++ b/schema/crdb/6.0.0/up09.sql @@ -0,0 +1,10 @@ +/* + * Because this is an offline update, the system comes back up with no active + * VMMs. Ensure all active Propolis IDs are cleared. This guileless approach + * gets planned as a full table scan, so explicitly (but temporarily) allow + * those. + */ + +set disallow_full_table_scans = off; +UPDATE omicron.public.instance SET active_propolis_id = NULL; +set disallow_full_table_scans = on; diff --git a/schema/crdb/README.adoc b/schema/crdb/README.adoc index ef96571d00..c15b51e374 100644 --- a/schema/crdb/README.adoc +++ b/schema/crdb/README.adoc @@ -62,7 +62,7 @@ Process: after your update is applied. Don't forget to update the version field of `db_metadata` at the bottom of the file! ** If necessary, do the same thing for `schema/crdb/dbwipe.sql`. -* Update Nexus's idea of the latest schema, by updating it's `SCHEMA_VERSION` to +* Update Nexus's idea of the latest schema, by updating its `SCHEMA_VERSION` to `NEW_VERSION` within `nexus/db-model/src/schema.rs`. SQL Validation, via Automated Tests: @@ -70,3 +70,65 @@ SQL Validation, via Automated Tests: * The `SCHEMA_VERSION` matches the version used in `dbinit.sql` * The combination of all `up.sql` files results in the same schema as `dbinit.sql` * All `up.sql` files can be applied twice without error + +==== Handling common schema changes + +CockroachDB's schema includes a description of all of the database's CHECK +constraints. If a CHECK constraint is anonymous (i.e. it is written simply as +`CHECK ` and not `CONSTRAINT CHECK expression`), CRDB +assigns it a name based on the table and column to which the constraint applies. +The challenge is that CRDB identifies tables and columns using opaque +identifiers whose values depend on the order in which tables and views were +defined in the current database. This means that adding, removing, or renaming +objects needs to be done carefully to preserve the relative ordering of objects +in new databases created by `dbinit.sql` and upgraded databases created by +applying `up.sql` transformations. + +===== Adding new columns with constraints + +Strongly consider naming new constraints (`CONSTRAINT `) to +avoid the problems with anonymous constraints described above. + +===== Adding tables and views + +New tables and views must be added to the end of `dbinit.sql` so that the order +of preceding `CREATE` statements is left unchanged. If your changes fail the +`CHECK` constraints test and you get a constraint name diff like this... + +``` +NamedSqlValue { + column: "constraint_name", + value: Some( + String( +< "4101115737_149_10_not_null", +> "4101115737_148_10_not_null", +``` + +...then you've probably inadvertently added a table or view in the wrong place. + +==== Adding new source tables to an existing view + +An upgrade can add a new table and then use a `CREATE OR REPLACE VIEW` statement +to make an existing view depend on that table. To do this in `dbinit.sql` while +maintaining table and view ordering, use `CREATE VIEW` to create a "placeholder" +view in the correct position, then add the table to the bottom of `dbinit.sql` +and use `CREATE OR REPLACE VIEW` to "fill out" the placeholder definition to +refer to the new table. (You may need to do the `CREATE OR REPLACE VIEW` in a +separate transaction from the original `CREATE VIEW`.) + +Note that `CREATE OR REPLACE VIEW` requires that the new view maintain all of +the columns of the old view with the same type and same order (though the query +used to populate them can change. See +https://www.postgresql.org/docs/15/sql-createview.html. + +==== Renaming columns + +Idempotently renaming existing columns is unfortunately not possible in our +current database configuration. (Postgres doesn't support the use of an `IF +EXISTS` qualifier on an `ALTER TABLE RENAME COLUMN` statement, and the version +of CockroachDB we use at this writing doesn't support the use of user-defined +functions as a workaround.) + +An (imperfect) workaround is to use the `#[diesel(column_name = foo)]` attribute +in Rust code to preserve the existing name of a column in the database while +giving its corresponding struct field a different, more meaningful name. diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index a62cbae5ea..2b06e4cbd6 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -834,44 +834,20 @@ CREATE TABLE IF NOT EXISTS omicron.public.instance ( /* user data for instance initialization systems (e.g. cloud-init) */ user_data BYTES NOT NULL, - /* - * TODO Would it make sense for the runtime state to live in a separate - * table? - */ - /* Runtime state */ + /* The state of the instance when it has no active VMM. */ state omicron.public.instance_state NOT NULL, time_state_updated TIMESTAMPTZ NOT NULL, state_generation INT NOT NULL, - /* - * Sled where the VM is currently running, if any. Note that when we - * support live migration, there may be multiple sleds associated with - * this Instance, but only one will be truly active. Still, consumers of - * this information should consider whether they also want to know the other - * sleds involved in the migration. - */ - active_sled_id UUID, - /* Identifies the underlying propolis-server backing the instance. */ - active_propolis_id UUID NOT NULL, - active_propolis_ip INET, + /* FK into `vmm` for the Propolis server that's backing this instance. */ + active_propolis_id UUID, - /* Identifies the target propolis-server during a migration of the instance. */ + /* FK into `vmm` for the migration target Propolis server, if one exists. */ target_propolis_id UUID, - /* - * Identifies an ongoing migration for this instance. - */ + /* Identifies any ongoing migration for this instance. */ migration_id UUID, - /* - * A generation number protecting information about the "location" of a - * running instance: its active server ID, Propolis ID and IP, and migration - * information. This is used for mutual exclusion (to allow only one - * migration to proceed at a time) and to coordinate state changes when a - * migration finishes. - */ - propolis_generation INT NOT NULL, - /* Instance configuration */ ncpus INT NOT NULL, memory INT NOT NULL, @@ -886,42 +862,23 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_instance_by_project ON omicron.public.i ) WHERE time_deleted IS NULL; --- Allow looking up instances by server. This is particularly --- useful for resource accounting within a sled. -CREATE UNIQUE INDEX IF NOT EXISTS lookup_instance_by_sled ON omicron.public.instance ( - active_sled_id, - id -) WHERE - time_deleted IS NULL; - /* * A special view of an instance provided to operators for insights into what's running * on a sled. + * + * This view requires the VMM table, which doesn't exist yet, so create a + * "placeholder" view here and replace it with the full view once the table is + * defined. See the README for more context. */ -CREATE VIEW IF NOT EXISTS omicron.public.sled_instance +CREATE VIEW IF NOT EXISTS omicron.public.sled_instance AS SELECT - instance.id, - instance.name, - silo.name as silo_name, - project.name as project_name, - instance.active_sled_id, - instance.time_created, - instance.time_modified, - instance.migration_id, - instance.ncpus, - instance.memory, - instance.state + instance.id FROM omicron.public.instance AS instance - JOIN omicron.public.project AS project ON - instance.project_id = project.id - JOIN omicron.public.silo AS silo ON - project.silo_id = silo.id WHERE instance.time_deleted IS NULL; - /* * Guest-Visible, Virtual Disks */ @@ -2543,8 +2500,13 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_address_config ( PRIMARY KEY (port_settings_id, address, interface_name) ); +/* + * The `sled_instance` view's definition needs to be modified in a separate + * transaction from the transaction that created it. + */ -/*******************************************************************/ +COMMIT; +BEGIN; /* * Metadata for the schema itself. This version number isn't great, as there's @@ -2580,4 +2542,52 @@ INSERT INTO omicron.public.db_metadata ( ( TRUE, NOW(), NOW(), '6.0.0', NULL) ON CONFLICT DO NOTHING; + + +-- Per-VMM state. +CREATE TABLE IF NOT EXISTS omicron.public.vmm ( + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + instance_id UUID NOT NULL, + state omicron.public.instance_state NOT NULL, + time_state_updated TIMESTAMPTZ NOT NULL, + state_generation INT NOT NULL, + sled_id UUID NOT NULL, + propolis_ip INET NOT NULL +); + +/* + * A special view of an instance provided to operators for insights into what's + * running on a sled. + * + * This view replaces the placeholder `sled_instance` view defined above. Any + * columns in the placeholder must appear in the replacement in the same order + * and with the same types they had in the placeholder. + */ + +CREATE OR REPLACE VIEW omicron.public.sled_instance +AS SELECT + instance.id, + instance.name, + silo.name as silo_name, + project.name as project_name, + vmm.sled_id as active_sled_id, + instance.time_created, + instance.time_modified, + instance.migration_id, + instance.ncpus, + instance.memory, + vmm.state +FROM + omicron.public.instance AS instance + JOIN omicron.public.project AS project ON + instance.project_id = project.id + JOIN omicron.public.silo AS silo ON + project.silo_id = silo.id + JOIN omicron.public.vmm AS vmm ON + instance.active_propolis_id = vmm.id +WHERE + instance.time_deleted IS NULL AND vmm.time_deleted IS NULL; + COMMIT; diff --git a/sled-agent/src/common/instance.rs b/sled-agent/src/common/instance.rs index 0f7b91e56b..9e285840e0 100644 --- a/sled-agent/src/common/instance.rs +++ b/sled-agent/src/common/instance.rs @@ -5,12 +5,65 @@ //! Describes the states of VM instances. use crate::params::InstanceMigrationSourceParams; -use chrono::Utc; +use chrono::{DateTime, Utc}; use omicron_common::api::external::InstanceState as ApiInstanceState; -use omicron_common::api::internal::nexus::InstanceRuntimeState; +use omicron_common::api::internal::nexus::{ + InstanceRuntimeState, SledInstanceState, VmmRuntimeState, +}; use propolis_client::api::{ - InstanceState as PropolisInstanceState, InstanceStateMonitorResponse, + InstanceState as PropolisApiState, InstanceStateMonitorResponse, }; +use uuid::Uuid; + +/// The instance and VMM state that sled agent maintains on a per-VMM basis. +#[derive(Clone, Debug)] +pub struct InstanceStates { + instance: InstanceRuntimeState, + vmm: VmmRuntimeState, + propolis_id: Uuid, +} + +/// Newtype to allow conversion from Propolis API states (returned by the +/// Propolis state monitor) to Nexus VMM states. +#[derive(Clone, Copy, Debug)] +pub(crate) struct PropolisInstanceState(PropolisApiState); + +impl From for PropolisInstanceState { + fn from(value: PropolisApiState) -> Self { + Self(value) + } +} + +impl From for ApiInstanceState { + fn from(value: PropolisInstanceState) -> Self { + use propolis_client::api::InstanceState as State; + match value.0 { + // Nexus uses the VMM state as the externally-visible instance state + // when an instance has an active VMM. A Propolis that is "creating" + // its virtual machine objects is "starting" from the external API's + // perspective. + State::Creating | State::Starting => ApiInstanceState::Starting, + State::Running => ApiInstanceState::Running, + State::Stopping => ApiInstanceState::Stopping, + // A Propolis that is stopped but not yet destroyed should still + // appear to be Stopping from an external API perspective, since + // they cannot be restarted yet. Instances become logically Stopped + // once Propolis reports that the VM is Destroyed (see below). + State::Stopped => ApiInstanceState::Stopping, + State::Rebooting => ApiInstanceState::Rebooting, + State::Migrating => ApiInstanceState::Migrating, + State::Repairing => ApiInstanceState::Repairing, + State::Failed => ApiInstanceState::Failed, + // Nexus needs to learn when a VM has entered the "destroyed" state + // so that it can release its resource reservation. When this + // happens, this module also clears the active VMM ID from the + // instance record, which will accordingly set the Nexus-owned + // instance state to Stopped, preventing this state from being used + // as an externally-visible instance state. + State::Destroyed => ApiInstanceState::Destroyed, + } + } +} /// Describes the status of the migration identified in an instance's runtime /// state as it relates to any migration status information reported by the @@ -21,30 +74,12 @@ pub enum ObservedMigrationStatus { /// progress. NoMigration, - /// Propolis thinks a migration is in progress, but its migration ID does - /// not agree with the instance's current runtime state: either the current - /// runtime state has no ID, or Propolis has an older ID than sled agent - /// does because a newer migration has begun (see below). - /// - /// This is expected in the following scenarios: - /// - /// - Propolis was initialized via migration in, after which Nexus cleared - /// the instance's migration IDs. - /// - Propolis was initialized via migration in, and the instance is about - /// to migrate again. Propolis will have the old ID (from the migration - /// in) while the instance runtime state has the new ID (from the pending - /// migration out). - MismatchedId, - - /// Either: - /// - /// - The instance's runtime state contains a migration ID, but Propolis did - /// not report any migration was in progress, or - /// - Propolis reported that the active migration is not done yet. - /// - /// The first case occurs when the current instance is queued to be a - /// migration source, but its Propolis changed state before any migration - /// request reached that Propolis. + /// The instance has a migration ID, but Propolis either has no migration ID + /// or a different ID from this one (possible if the Propolis was + /// initialized via migration in). + Pending, + + /// Propolis reported that migration isn't done yet. InProgress, /// Propolis reported that the migration completed successfully. @@ -56,17 +91,16 @@ pub enum ObservedMigrationStatus { /// The information observed by the instance's Propolis state monitor. #[derive(Clone, Copy, Debug)] -pub struct ObservedPropolisState { +pub(crate) struct ObservedPropolisState { /// The state reported by Propolis's instance state monitor API. - /// - /// Note that this API allows transitions to be missed (if multiple - /// transitions occur between calls to the monitor, only the most recent - /// state is reported). - pub instance_state: PropolisInstanceState, + pub vmm_state: PropolisInstanceState, /// Information about whether the state observer queried migration status at /// all and, if so, what response it got from Propolis. pub migration_status: ObservedMigrationStatus, + + /// The approximate time at which this observation was made. + pub time: DateTime, } impl ObservedPropolisState { @@ -74,11 +108,11 @@ impl ObservedPropolisState { /// runtime state and an instance state monitor response received from /// Propolis. pub fn new( - runtime_state: &InstanceRuntimeState, + instance_runtime: &InstanceRuntimeState, propolis_state: &InstanceStateMonitorResponse, ) -> Self { let migration_status = - match (runtime_state.migration_id, &propolis_state.migration) { + match (instance_runtime.migration_id, &propolis_state.migration) { // If the runtime state and Propolis state agree that there's // a migration in progress, and they agree on its ID, the // Propolis migration state determines the migration status. @@ -97,22 +131,33 @@ impl ObservedPropolisState { } } - // If the migration IDs don't match, or Propolis thinks a - // migration is in progress but the instance's runtime state - // does not, report the mismatch. - (_, Some(_)) => ObservedMigrationStatus::MismatchedId, + // If both sides have a migration ID, but the IDs don't match, + // assume the instance's migration ID is newer. This can happen + // if Propolis was initialized via migration in and has not yet + // been told to migrate out. + (Some(_), Some(_)) => ObservedMigrationStatus::Pending, + + // If only Propolis has a migration ID, assume it was from a + // prior migration in and report that no migration is in + // progress. This could be improved with propolis#508. + (None, Some(_)) => ObservedMigrationStatus::NoMigration, // A migration source's migration IDs get set before its // Propolis actually gets asked to migrate, so it's possible for // the runtime state to contain an ID while the Propolis has // none, in which case the migration is pending. - (Some(_), None) => ObservedMigrationStatus::InProgress, + (Some(_), None) => ObservedMigrationStatus::Pending, // If neither side has a migration ID, then there's clearly no // migration. (None, None) => ObservedMigrationStatus::NoMigration, }; - Self { instance_state: propolis_state.state, migration_status } + + Self { + vmm_state: PropolisInstanceState(propolis_state.state), + migration_status, + time: Utc::now(), + } } } @@ -120,218 +165,261 @@ impl ObservedPropolisState { /// a subset of the instance states Nexus knows about: the Creating and /// Destroyed states are reserved for Nexus to use for instances that are being /// created for the very first time or have been explicitly deleted. -pub enum PublishedInstanceState { - Starting, - Running, +pub enum PublishedVmmState { Stopping, - Stopped, Rebooting, - Migrating, - Repairing, - Failed, } -impl From for PublishedInstanceState { - fn from(value: PropolisInstanceState) -> Self { +impl From for ApiInstanceState { + fn from(value: PublishedVmmState) -> Self { match value { - // From an external perspective, the instance has already been - // created. Creating the propolis instance is an internal detail and - // happens every time we start the instance, so we map it to - // "Starting" here. - PropolisInstanceState::Creating - | PropolisInstanceState::Starting => { - PublishedInstanceState::Starting - } - PropolisInstanceState::Running => PublishedInstanceState::Running, - PropolisInstanceState::Stopping => PublishedInstanceState::Stopping, - PropolisInstanceState::Stopped => PublishedInstanceState::Stopped, - PropolisInstanceState::Rebooting => { - PublishedInstanceState::Rebooting - } - PropolisInstanceState::Migrating => { - PublishedInstanceState::Migrating - } - PropolisInstanceState::Repairing => { - PublishedInstanceState::Repairing - } - PropolisInstanceState::Failed => PublishedInstanceState::Failed, - // NOTE: This is a bit of an odd one - we intentionally do *not* - // translate the "destroyed" propolis state to the destroyed instance - // API state. - // - // When a propolis instance reports that it has been destroyed, - // this does not necessarily mean the customer-visible instance - // should be torn down. Instead, it implies that the Propolis service - // should be stopped, but the VM could be allocated to a different - // machine. - PropolisInstanceState::Destroyed => PublishedInstanceState::Stopped, + PublishedVmmState::Stopping => ApiInstanceState::Stopping, + PublishedVmmState::Rebooting => ApiInstanceState::Rebooting, } } } -impl From for ApiInstanceState { - fn from(value: PublishedInstanceState) -> Self { - match value { - PublishedInstanceState::Starting => ApiInstanceState::Starting, - PublishedInstanceState::Running => ApiInstanceState::Running, - PublishedInstanceState::Stopping => ApiInstanceState::Stopping, - PublishedInstanceState::Stopped => ApiInstanceState::Stopped, - PublishedInstanceState::Rebooting => ApiInstanceState::Rebooting, - PublishedInstanceState::Migrating => ApiInstanceState::Migrating, - PublishedInstanceState::Repairing => ApiInstanceState::Repairing, - PublishedInstanceState::Failed => ApiInstanceState::Failed, - } - } +/// The possible roles a VMM can have vis-a-vis an instance. +#[derive(Clone, Copy, Debug, PartialEq)] +enum PropolisRole { + /// The VMM is its instance's current active VMM. + Active, + + /// The VMM is its instance's migration target VMM. + MigrationTarget, + + /// The instance does not refer to this VMM (but it may have done so in the + /// past). + Retired, } /// Action to be taken on behalf of state transition. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum Action { - /// Update the VM state to cause it to run. - Run, - /// Update the VM state to cause it to stop. - Stop, - /// Invoke a reboot of the VM. - Reboot, /// Terminate the VM and associated service. Destroy, } -/// A wrapper around an instance's current state, represented as a Nexus -/// `InstanceRuntimeState`. The externally-visible instance state in this -/// structure is mostly changed when the instance's Propolis state changes. -#[derive(Clone, Debug)] -pub struct InstanceStates { - current: InstanceRuntimeState, -} - impl InstanceStates { - pub fn new(current: InstanceRuntimeState) -> Self { - InstanceStates { current } + pub fn new( + instance: InstanceRuntimeState, + vmm: VmmRuntimeState, + propolis_id: Uuid, + ) -> Self { + InstanceStates { instance, vmm, propolis_id } + } + + pub fn instance(&self) -> &InstanceRuntimeState { + &self.instance + } + + pub fn vmm(&self) -> &VmmRuntimeState { + &self.vmm } - /// Returns the current instance state. - pub fn current(&self) -> &InstanceRuntimeState { - &self.current + pub fn propolis_id(&self) -> Uuid { + self.propolis_id } - /// Returns the current instance state. - pub fn current_mut(&mut self) -> &mut InstanceRuntimeState { - &mut self.current + /// Creates a `SledInstanceState` structure containing the entirety of this + /// structure's runtime state. This requires cloning; for simple read access + /// use the `instance` or `vmm` accessors instead. + pub fn sled_instance_state(&self) -> SledInstanceState { + SledInstanceState { + instance_state: self.instance.clone(), + vmm_state: self.vmm.clone(), + propolis_id: self.propolis_id, + } } /// Update the known state of an instance based on an observed state from /// Propolis. - pub fn apply_propolis_observation( + pub(crate) fn apply_propolis_observation( &mut self, observed: &ObservedPropolisState, ) -> Option { - // The state after this transition will be published to Nexus, so some - // care is required around migration to ensure that Nexus's instance - // state remains consistent even in the face of racing updates from a - // migration source and a migration target. The possibilities are as - // follows: - // - // 1. The current migration succeeded. - // 1a. Migration source: ideally this case would pass control to the - // target explicitly, but there currently isn't enough information - // to do that (the source doesn't know the target's sled ID or - // Propolis IP), so just let the target deal with updating - // everything. - // - // This is the one case in which this routine explicitly *should - // not* transition the current state (to avoid having a "stopped" - // state reach Nexus before the target takes control of the state - // machine). - // 1b. Migration target: Signal that migration is done by bumping the - // Propolis generation number and clearing the migration ID and - // destination Propolis ID from the instance record. - // 2. The current migration failed. - // 2a. Migration source: The source is running now. Clear the - // migration IDs, bump the Propolis generation number, and publish - // the updated state, ending the migration. - // 2b. Migration target: The target has failed and control of the - // instance remains with the source. Don't update the Propolis - // generation number. Updating state is OK here because migration - // targets can't update Nexus instance states without changing the - // Propolis generation. - // 3. No migration is ongoing, or the migration ID in the instance - // record doesn't line up with the putatively ongoing migration. Just - // update state normally in this case; whichever sled has the current - // Propolis generation will have its update applied. - // - // There is an additional exceptional case here: when an instance stops, - // its migration IDs should be cleared so that it can migrate when it is - // started again. If the VM is in a terminal state, and this is *not* - // case 1a above (i.e. the Propolis is stopping because the instance - // migrated out), clear any leftover migration IDs. - // - // TODO(#2315): Terminal-state cleanup should also clear an instance's - // sled assignment and Propolis ID, but that requires Nexus work to - // repopulate these when the instance starts again. - let action = if matches!( - observed.instance_state, - PropolisInstanceState::Destroyed | PropolisInstanceState::Failed - ) { - Some(Action::Destroy) - } else { - None - }; + let vmm_gone = matches!( + observed.vmm_state.0, + PropolisApiState::Destroyed | PropolisApiState::Failed + ); + + // Apply this observation to the VMM record. It is safe to apply the + // Destroyed state directly here because this routine ensures that if + // this VMM is active, it will be retired and an appropriate + // non-Destroyed state applied to the instance itself. + self.vmm.state = observed.vmm_state.into(); + self.vmm.gen = self.vmm.gen.next(); + self.vmm.time_updated = observed.time; - let next_state = PublishedInstanceState::from(observed.instance_state); + // Update the instance record to reflect the result of any completed + // migration. match observed.migration_status { - // Case 3: Update normally if there is no migration in progress or - // the current migration is unrecognized or in flight. - ObservedMigrationStatus::NoMigration - | ObservedMigrationStatus::MismatchedId - | ObservedMigrationStatus::InProgress => { - self.transition(next_state); - } + ObservedMigrationStatus::Succeeded => match self.propolis_role() { + // This is a successful migration out. Point the instance to the + // target VMM, but don't clear migration IDs; let the target do + // that so that the instance will continue to appear to be + // migrating until it is safe to migrate again. + PropolisRole::Active => { + self.switch_propolis_id_to_target(observed.time); + + assert_eq!(self.propolis_role(), PropolisRole::Retired); + } - // Case 1: Migration succeeded. Only update the instance record if - // this is a migration target. - // - // Calling `is_migration_target` is safe here because the instance - // must have had a migration ID in its record to have inferred that - // an ongoing migration succeeded. - ObservedMigrationStatus::Succeeded => { - if self.is_migration_target() { - self.transition(next_state); - self.clear_migration_ids(); - } else { - // Case 1a: Short-circuit without touching the instance - // record. - return action; + // This is a successful migration in. Point the instance to the + // target VMM and clear migration IDs so that another migration + // in can begin. Propolis will continue reporting that this + // migration was successful, but because its ID has been + // discarded the observed migration status will change from + // Succeeded to NoMigration. + // + // Note that these calls increment the instance's generation + // number twice. This is by design and allows the target's + // migration-ID-clearing update to overtake the source's update. + PropolisRole::MigrationTarget => { + self.switch_propolis_id_to_target(observed.time); + self.clear_migration_ids(observed.time); + + assert_eq!(self.propolis_role(), PropolisRole::Active); } - } - // Case 2: Migration failed. Only update the instance record if this - // is a migration source. (Updating the target record is allowed, - // but still has to short-circuit so that the call to - // `clear_migration_ids` below is not reached.) - ObservedMigrationStatus::Failed => { - if self.is_migration_target() { - return action; - } else { - self.transition(next_state); - self.clear_migration_ids(); + // This is a migration source that previously reported success + // and removed itself from the active Propolis position. Don't + // touch the instance. + PropolisRole::Retired => {} + }, + ObservedMigrationStatus::Failed => match self.propolis_role() { + // This is a failed migration out. CLear migration IDs so that + // Nexus can try again. + PropolisRole::Active => { + self.clear_migration_ids(observed.time); } + + // This is a failed migration in. Leave the migration IDs alone + // so that the migration won't appear to have concluded until + // the source is ready to start a new one. + PropolisRole::MigrationTarget => {} + + // This VMM was part of a failed migration and was subsequently + // removed from the instance record entirely. There's nothing to + // update. + PropolisRole::Retired => {} + }, + ObservedMigrationStatus::NoMigration + | ObservedMigrationStatus::InProgress + | ObservedMigrationStatus::Pending => {} + } + + // If this Propolis has exited, tear down its zone. If it was in the + // active position, immediately retire any migration that might have + // been pending and clear the active Propolis ID so that the instance + // can start somewhere else. + // + // N.B. It is important to refetch the current Propolis role here, + // because it might have changed in the course of dealing with a + // completed migration. (In particular, if this VMM is gone because + // it was the source of a successful migration out, control has + // been transferred to the target, and what was once an active VMM + // is now retired.) + if vmm_gone { + if self.propolis_role() == PropolisRole::Active { + self.clear_migration_ids(observed.time); + self.retire_active_propolis(observed.time); + } + Some(Action::Destroy) + } else { + None + } + } + + /// Yields the role that this structure's VMM has given the structure's + /// current instance state. + fn propolis_role(&self) -> PropolisRole { + if let Some(active_id) = self.instance.propolis_id { + if active_id == self.propolis_id { + return PropolisRole::Active; } } - if matches!(action, Some(Action::Destroy)) { - self.clear_migration_ids(); + if let Some(dst_id) = self.instance.dst_propolis_id { + if dst_id == self.propolis_id { + return PropolisRole::MigrationTarget; + } } - action + PropolisRole::Retired } - // Transitions to a new InstanceState value, updating the timestamp and - // generation number. - pub(crate) fn transition(&mut self, next: PublishedInstanceState) { - self.current.run_state = next.into(); - self.current.gen = self.current.gen.next(); - self.current.time_updated = Utc::now(); + /// Sets the no-VMM fallback state of the current instance to reflect the + /// state of its terminated VMM and clears the instance's current Propolis + /// ID. Note that this routine does not touch any migration IDs. + /// + /// This should only be called by the state block for an active VMM and only + /// when that VMM is in a terminal state (Destroyed or Failed). + fn retire_active_propolis(&mut self, now: DateTime) { + assert!(self.propolis_role() == PropolisRole::Active); + + self.instance.propolis_id = None; + self.instance.gen = self.instance.gen.next(); + self.instance.time_updated = now; + } + + /// Moves the instance's destination Propolis ID into the current active + /// position and updates the generation number, but does not clear the + /// destination ID or the active migration ID. This promotes a migration + /// target VMM into the active position without actually allowing a new + /// migration to begin. + /// + /// This routine should only be called when + /// `instance.dst_propolis_id.is_some()`. + fn switch_propolis_id_to_target(&mut self, now: DateTime) { + assert!(self.instance.dst_propolis_id.is_some()); + + self.instance.propolis_id = self.instance.dst_propolis_id; + self.instance.gen = self.instance.gen.next(); + self.instance.time_updated = now; + } + + /// Forcibly transitions this instance's VMM into the specified `next` + /// state and updates its generation number. + pub(crate) fn transition_vmm( + &mut self, + next: PublishedVmmState, + now: DateTime, + ) { + self.vmm.state = next.into(); + self.vmm.gen = self.vmm.gen.next(); + self.vmm.time_updated = now; + } + + /// Updates the state of this instance in response to a rude termination of + /// its Propolis zone, marking the VMM as destroyed and applying any + /// consequent state updates. + /// + /// # Synchronization + /// + /// A caller who is rudely terminating a Propolis zone must hold locks + /// sufficient to ensure that no other Propolis observations arrive in the + /// transaction that terminates the zone and then calls this function. + /// + /// TODO(#4004): This routine works by synthesizing a Propolis state change + /// that says "this Propolis is destroyed and its active migration failed." + /// If this conflicts with the actual Propolis state--e.g., if the + /// underlying Propolis was destroyed but migration *succeeded*--the + /// instance's state in Nexus may become inconsistent. This routine should + /// therefore only be invoked by callers who know that an instance is not + /// migrating. + pub(crate) fn terminate_rudely(&mut self) { + let fake_observed = ObservedPropolisState { + vmm_state: PropolisInstanceState(PropolisApiState::Destroyed), + migration_status: if self.instance.migration_id.is_some() { + ObservedMigrationStatus::Failed + } else { + ObservedMigrationStatus::NoMigration + }, + time: Utc::now(), + }; + + self.apply_propolis_observation(&fake_observed); } /// Sets or clears this instance's migration IDs and advances its Propolis @@ -339,24 +427,27 @@ impl InstanceStates { pub(crate) fn set_migration_ids( &mut self, ids: &Option, + now: DateTime, ) { if let Some(ids) = ids { - self.current.migration_id = Some(ids.migration_id); - self.current.dst_propolis_id = Some(ids.dst_propolis_id); + self.instance.migration_id = Some(ids.migration_id); + self.instance.dst_propolis_id = Some(ids.dst_propolis_id); } else { - self.current.migration_id = None; - self.current.dst_propolis_id = None; + self.instance.migration_id = None; + self.instance.dst_propolis_id = None; } - self.current.propolis_gen = self.current.propolis_gen.next(); + self.instance.gen = self.instance.gen.next(); + self.instance.time_updated = now; } /// Unconditionally clears the instance's migration IDs and advances its /// Propolis generation. Not public; used internally to conclude migrations. - fn clear_migration_ids(&mut self) { - self.current.migration_id = None; - self.current.dst_propolis_id = None; - self.current.propolis_gen = self.current.propolis_gen.next(); + fn clear_migration_ids(&mut self, now: DateTime) { + self.instance.migration_id = None; + self.instance.dst_propolis_id = None; + self.instance.gen = self.instance.gen.next(); + self.instance.time_updated = now; } /// Returns true if the migration IDs in this instance are already set as they @@ -384,15 +475,15 @@ impl InstanceStates { // A simple less-than check allows the migration to sled 3 to proceed // even though the most-recently-expressed intent to migrate put the // instance on sled 1. - if old_runtime.propolis_gen.next() != self.current.propolis_gen { + if old_runtime.gen.next() != self.instance.gen { return false; } - match (self.current.migration_id, migration_ids) { + match (self.instance.migration_id, migration_ids) { // If the migration ID is already set, and this is a request to set // IDs, the records match if the relevant IDs match. (Some(current_migration_id), Some(ids)) => { - let current_dst_id = self.current.dst_propolis_id.expect( + let current_dst_id = self.instance.dst_propolis_id.expect( "migration ID and destination ID must be set together", ); @@ -402,23 +493,12 @@ impl InstanceStates { // If the migration ID is already cleared, and this is a request to // clear IDs, the records match. (None, None) => { - assert!(self.current.dst_propolis_id.is_none()); + assert!(self.instance.dst_propolis_id.is_none()); true } _ => false, } } - - /// Indicates whether this instance incarnation is a migration source or - /// target by comparing the instance's current active Propolis ID with its - /// migration destination ID. - /// - /// # Panics - /// - /// Panics if the instance has no destination Propolis ID set. - fn is_migration_target(&self) -> bool { - self.current.propolis_id == self.current.dst_propolis_id.unwrap() - } } #[cfg(test)] @@ -428,43 +508,45 @@ mod test { use crate::params::InstanceMigrationSourceParams; use chrono::Utc; - use omicron_common::api::external::{ - ByteCount, Generation, InstanceCpuCount, InstanceState as State, - }; + use omicron_common::api::external::Generation; use omicron_common::api::internal::nexus::InstanceRuntimeState; use propolis_client::api::InstanceState as Observed; use uuid::Uuid; fn make_instance() -> InstanceStates { - InstanceStates::new(InstanceRuntimeState { - run_state: State::Creating, - sled_id: Uuid::new_v4(), - propolis_id: Uuid::new_v4(), + let propolis_id = Uuid::new_v4(); + let now = Utc::now(); + let instance = InstanceRuntimeState { + propolis_id: Some(propolis_id), dst_propolis_id: None, - propolis_addr: None, migration_id: None, - propolis_gen: Generation::new(), - ncpus: InstanceCpuCount(2), - memory: ByteCount::from_mebibytes_u32(512), - hostname: "myvm".to_string(), gen: Generation::new(), - time_updated: Utc::now(), - }) + time_updated: now, + }; + + let vmm = VmmRuntimeState { + state: ApiInstanceState::Starting, + gen: Generation::new(), + time_updated: now, + }; + + InstanceStates::new(instance, vmm, propolis_id) } fn make_migration_source_instance() -> InstanceStates { let mut state = make_instance(); - state.current.run_state = State::Migrating; - state.current.migration_id = Some(Uuid::new_v4()); - state.current.dst_propolis_id = Some(Uuid::new_v4()); + state.vmm.state = ApiInstanceState::Migrating; + state.instance.migration_id = Some(Uuid::new_v4()); + state.instance.dst_propolis_id = Some(Uuid::new_v4()); state } fn make_migration_target_instance() -> InstanceStates { let mut state = make_instance(); - state.current.run_state = State::Migrating; - state.current.migration_id = Some(Uuid::new_v4()); - state.current.dst_propolis_id = Some(state.current.propolis_id); + state.vmm.state = ApiInstanceState::Migrating; + state.instance.migration_id = Some(Uuid::new_v4()); + state.propolis_id = Uuid::new_v4(); + state.instance.dst_propolis_id = Some(state.propolis_id); state } @@ -472,124 +554,239 @@ mod test { propolis_state: PropolisInstanceState, ) -> ObservedPropolisState { ObservedPropolisState { - instance_state: propolis_state, + vmm_state: propolis_state, migration_status: ObservedMigrationStatus::NoMigration, + time: Utc::now(), + } + } + + /// Checks to see if the instance state structures `prev` and `next` have a + /// difference that should produce a change in generation and asserts that + /// such a change occurred. + fn assert_state_change_has_gen_change( + prev: &InstanceStates, + next: &InstanceStates, + ) { + // The predicate under test below is "if an interesting field changed, + // then the generation number changed." Testing the contrapositive is a + // little nicer because the assertion that trips identifies exactly + // which field changed without updating the generation number. + // + // The else branch tests the converse to make sure the generation number + // does not update unexpectedly. While this won't cause an important + // state update to be dropped, it can interfere with updates from other + // sleds that expect their own attempts to advance the generation number + // to cause new state to be recorded. + if prev.instance.gen == next.instance.gen { + assert_eq!(prev.instance.propolis_id, next.instance.propolis_id); + assert_eq!( + prev.instance.dst_propolis_id, + next.instance.dst_propolis_id + ); + assert_eq!(prev.instance.migration_id, next.instance.migration_id); + } else { + assert!( + (prev.instance.propolis_id != next.instance.propolis_id) + || (prev.instance.dst_propolis_id + != next.instance.dst_propolis_id) + || (prev.instance.migration_id + != next.instance.migration_id), + "prev: {:?}, next: {:?}", + prev, + next + ); + } + + // Propolis is free to publish no-op VMM state updates (e.g. when an + // in-progress migration's state changes but the migration is not yet + // complete), so don't test the converse here. + if prev.vmm.gen == next.vmm.gen { + assert_eq!(prev.vmm.state, next.vmm.state); } } #[test] fn propolis_terminal_states_request_destroy_action() { for state in [Observed::Destroyed, Observed::Failed] { - let mut instance = make_instance(); - let original_instance = instance.clone(); - let requested_action = instance - .apply_propolis_observation(&make_observed_state(state)); + let mut instance_state = make_instance(); + let original_instance_state = instance_state.clone(); + let requested_action = instance_state + .apply_propolis_observation(&make_observed_state(state.into())); assert!(matches!(requested_action, Some(Action::Destroy))); - assert!(instance.current.gen > original_instance.current.gen); + assert!( + instance_state.instance.gen + > original_instance_state.instance.gen + ); } } #[test] fn destruction_after_migration_out_does_not_transition() { - let mut instance = make_migration_source_instance(); + let mut state = make_migration_source_instance(); + assert!(state.instance.dst_propolis_id.is_some()); + assert_ne!(state.instance.propolis_id, state.instance.dst_propolis_id); + + // After a migration succeeds, the source VM appears to stop but reports + // that the migration has succeeded. let mut observed = ObservedPropolisState { - instance_state: Observed::Stopping, + vmm_state: PropolisInstanceState(Observed::Stopping), migration_status: ObservedMigrationStatus::Succeeded, + time: Utc::now(), }; - let original = instance.clone(); - assert!(instance.apply_propolis_observation(&observed).is_none()); - assert_eq!(instance.current.gen, original.current.gen); - - observed.instance_state = Observed::Stopped; - assert!(instance.apply_propolis_observation(&observed).is_none()); - assert_eq!(instance.current.gen, original.current.gen); - - observed.instance_state = Observed::Destroyed; + // This transition should transfer control to the target VMM without + // actually marking the migration as completed. This advances the + // instance's state generation. + let prev = state.clone(); + assert!(state.apply_propolis_observation(&observed).is_none()); + assert_state_change_has_gen_change(&prev, &state); + assert!(state.instance.gen > prev.instance.gen); + assert_eq!( + state.instance.dst_propolis_id, + prev.instance.dst_propolis_id + ); + assert_eq!(state.instance.propolis_id, state.instance.dst_propolis_id); + assert!(state.instance.migration_id.is_some()); + + // Once a successful migration is observed, the VMM's state should + // continue to update, but the instance's state shouldn't change + // anymore. + let prev = state.clone(); + observed.vmm_state = PropolisInstanceState(Observed::Stopped); + assert!(state.apply_propolis_observation(&observed).is_none()); + assert_state_change_has_gen_change(&prev, &state); + assert_eq!(state.instance.gen, prev.instance.gen); + + // The Stopped state is translated internally to Stopping to prevent + // external viewers from perceiving that the instance is stopped before + // the VMM is fully retired. + assert_eq!(state.vmm.state, ApiInstanceState::Stopping); + assert!(state.vmm.gen > prev.vmm.gen); + + let prev = state.clone(); + observed.vmm_state = PropolisInstanceState(Observed::Destroyed); assert!(matches!( - instance.apply_propolis_observation(&observed), + state.apply_propolis_observation(&observed), Some(Action::Destroy) )); - assert_eq!(instance.current.gen, original.current.gen); + assert_state_change_has_gen_change(&prev, &state); + assert_eq!(state.instance.gen, prev.instance.gen); + assert_eq!(state.vmm.state, ApiInstanceState::Destroyed); + assert!(state.vmm.gen > prev.vmm.gen); } #[test] fn failure_after_migration_in_does_not_transition() { - let mut instance = make_migration_target_instance(); + let mut state = make_migration_target_instance(); + + // Failure to migrate into an instance should mark the VMM as destroyed + // but should not change the instance's migration IDs. let observed = ObservedPropolisState { - instance_state: Observed::Failed, + vmm_state: PropolisInstanceState(Observed::Failed), migration_status: ObservedMigrationStatus::Failed, + time: Utc::now(), }; - let original = instance.clone(); + let prev = state.clone(); assert!(matches!( - instance.apply_propolis_observation(&observed), + state.apply_propolis_observation(&observed), Some(Action::Destroy) )); - assert_eq!(instance.current.gen, original.current.gen); + assert_state_change_has_gen_change(&prev, &state); + assert_eq!(state.instance.gen, prev.instance.gen); + assert_eq!(state.vmm.state, ApiInstanceState::Failed); + assert!(state.vmm.gen > prev.vmm.gen); + } + + // Verifies that the rude-termination state change doesn't update the + // instance record if the VMM under consideration is a migration target. + // + // The live migration saga relies on this property for correctness (it needs + // to know that unwinding its "create destination VMM" step will not produce + // an updated instance record). + #[test] + fn rude_terminate_of_migration_target_does_not_transition_instance() { + let mut state = make_migration_target_instance(); + assert_eq!(state.propolis_role(), PropolisRole::MigrationTarget); + + let prev = state.clone(); + state.terminate_rudely(); + + assert_state_change_has_gen_change(&prev, &state); + assert_eq!(state.instance.gen, prev.instance.gen); } #[test] fn migration_out_after_migration_in() { - let mut instance = make_migration_target_instance(); + let mut state = make_migration_target_instance(); let mut observed = ObservedPropolisState { - instance_state: Observed::Running, + vmm_state: PropolisInstanceState(Observed::Running), migration_status: ObservedMigrationStatus::Succeeded, + time: Utc::now(), }; // The transition into the Running state on the migration target should // take over for the source, updating the Propolis generation. - let prev = instance.clone(); - assert!(instance.apply_propolis_observation(&observed).is_none()); - assert!(instance.current.migration_id.is_none()); - assert!(instance.current.dst_propolis_id.is_none()); - assert!(instance.current.gen > prev.current.gen); - assert!(instance.current.propolis_gen > prev.current.propolis_gen); + let prev = state.clone(); + assert!(state.apply_propolis_observation(&observed).is_none()); + assert_state_change_has_gen_change(&prev, &state); + assert!(state.instance.migration_id.is_none()); + assert!(state.instance.dst_propolis_id.is_none()); + assert!(state.instance.gen > prev.instance.gen); + assert_eq!(state.vmm.state, ApiInstanceState::Running); + assert!(state.vmm.gen > prev.vmm.gen); // Pretend Nexus set some new migration IDs. - let prev = instance.clone(); - instance.set_migration_ids(&Some(InstanceMigrationSourceParams { - migration_id: Uuid::new_v4(), - dst_propolis_id: Uuid::new_v4(), - })); - assert!(instance.current.propolis_gen > prev.current.propolis_gen); - - // Mark that the new migration out is in progress. - let prev = instance.clone(); - observed.instance_state = Observed::Migrating; + let prev = state.clone(); + state.set_migration_ids( + &Some(InstanceMigrationSourceParams { + migration_id: Uuid::new_v4(), + dst_propolis_id: Uuid::new_v4(), + }), + Utc::now(), + ); + assert_state_change_has_gen_change(&prev, &state); + assert!(state.instance.gen > prev.instance.gen); + assert_eq!(state.vmm.gen, prev.vmm.gen); + + // Mark that the new migration out is in progress. This doesn't change + // anything in the instance runtime state, but does update the VMM state + // generation. + let prev = state.clone(); + observed.vmm_state = PropolisInstanceState(Observed::Migrating); observed.migration_status = ObservedMigrationStatus::InProgress; - assert!(instance.apply_propolis_observation(&observed).is_none()); + assert!(state.apply_propolis_observation(&observed).is_none()); + assert_state_change_has_gen_change(&prev, &state); assert_eq!( - instance.current.migration_id.unwrap(), - prev.current.migration_id.unwrap() + state.instance.migration_id.unwrap(), + prev.instance.migration_id.unwrap() ); assert_eq!( - instance.current.dst_propolis_id.unwrap(), - prev.current.dst_propolis_id.unwrap() + state.instance.dst_propolis_id.unwrap(), + prev.instance.dst_propolis_id.unwrap() ); - assert_eq!(instance.current.run_state, State::Migrating); - assert!(instance.current.gen > prev.current.gen); - assert_eq!(instance.current.propolis_gen, prev.current.propolis_gen); + assert_eq!(state.vmm.state, ApiInstanceState::Migrating); + assert!(state.vmm.gen > prev.vmm.gen); + assert_eq!(state.instance.gen, prev.instance.gen); // Propolis will publish that the migration succeeds before changing any - // state. Because this is now a successful migration source, the - // instance record is not updated. - observed.instance_state = Observed::Migrating; + // state. This should transfer control to the target but should not + // touch the migration ID (that is the new target's job). + let prev = state.clone(); + observed.vmm_state = PropolisInstanceState(Observed::Migrating); observed.migration_status = ObservedMigrationStatus::Succeeded; - let prev = instance.clone(); - assert!(instance.apply_propolis_observation(&observed).is_none()); - assert_eq!(instance.current.run_state, State::Migrating); - assert_eq!( - instance.current.migration_id.unwrap(), - prev.current.migration_id.unwrap() - ); + assert!(state.apply_propolis_observation(&observed).is_none()); + assert_state_change_has_gen_change(&prev, &state); + assert_eq!(state.vmm.state, ApiInstanceState::Migrating); + assert!(state.vmm.gen > prev.vmm.gen); + assert_eq!(state.instance.migration_id, prev.instance.migration_id); assert_eq!( - instance.current.dst_propolis_id.unwrap(), - prev.current.dst_propolis_id.unwrap() + state.instance.dst_propolis_id, + prev.instance.dst_propolis_id, ); - assert_eq!(instance.current.gen, prev.current.gen); - assert_eq!(instance.current.propolis_gen, prev.current.propolis_gen); + assert_eq!(state.instance.propolis_id, state.instance.dst_propolis_id); + assert!(state.instance.gen > prev.instance.gen); // The rest of the destruction sequence is covered by other tests. } @@ -607,51 +804,49 @@ mod test { dst_propolis_id: Uuid::new_v4(), }; - new_instance.set_migration_ids(&Some(migration_ids)); + new_instance.set_migration_ids(&Some(migration_ids), Utc::now()); assert!(new_instance.migration_ids_already_set( - old_instance.current(), + old_instance.instance(), &Some(migration_ids) )); // The IDs aren't already set if the new record has an ID that's // advanced from the old record by more than one generation. let mut newer_instance = new_instance.clone(); - newer_instance.current.propolis_gen = - newer_instance.current.propolis_gen.next(); + newer_instance.instance.gen = newer_instance.instance.gen.next(); assert!(!newer_instance.migration_ids_already_set( - old_instance.current(), + old_instance.instance(), &Some(migration_ids) )); // They also aren't set if the old generation has somehow equaled or // surpassed the current generation. - old_instance.current.propolis_gen = - old_instance.current.propolis_gen.next(); + old_instance.instance.gen = old_instance.instance.gen.next(); assert!(!new_instance.migration_ids_already_set( - old_instance.current(), + old_instance.instance(), &Some(migration_ids) )); // If the generation numbers are right, but either requested ID is not // present in the current instance, the requested IDs aren't set. old_instance = orig_instance; - new_instance.current.migration_id = Some(Uuid::new_v4()); + new_instance.instance.migration_id = Some(Uuid::new_v4()); assert!(!new_instance.migration_ids_already_set( - old_instance.current(), + old_instance.instance(), &Some(migration_ids) )); - new_instance.current.migration_id = Some(migration_ids.migration_id); - new_instance.current.dst_propolis_id = Some(Uuid::new_v4()); + new_instance.instance.migration_id = Some(migration_ids.migration_id); + new_instance.instance.dst_propolis_id = Some(Uuid::new_v4()); assert!(!new_instance.migration_ids_already_set( - old_instance.current(), + old_instance.instance(), &Some(migration_ids) )); - new_instance.current.migration_id = None; - new_instance.current.dst_propolis_id = None; + new_instance.instance.migration_id = None; + new_instance.instance.dst_propolis_id = None; assert!(!new_instance.migration_ids_already_set( - old_instance.current(), + old_instance.instance(), &Some(migration_ids) )); } diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 440ccb73ee..2ab8273e39 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -25,7 +25,7 @@ use illumos_utils::opte::params::{ }; use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::DiskRuntimeState; -use omicron_common::api::internal::nexus::InstanceRuntimeState; +use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::UpdateArtifactId; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -393,12 +393,20 @@ async fn instance_register( rqctx: RequestContext, path_params: Path, body: TypedBody, -) -> Result, HttpError> { +) -> Result, HttpError> { let sa = rqctx.context(); let instance_id = path_params.into_inner().instance_id; let body_args = body.into_inner(); Ok(HttpResponseOk( - sa.instance_ensure_registered(instance_id, body_args.initial).await?, + sa.instance_ensure_registered( + instance_id, + body_args.propolis_id, + body_args.hardware, + body_args.instance_runtime, + body_args.vmm_runtime, + body_args.propolis_addr, + ) + .await?, )) } @@ -440,7 +448,7 @@ async fn instance_put_migration_ids( rqctx: RequestContext, path_params: Path, body: TypedBody, -) -> Result, HttpError> { +) -> Result, HttpError> { let sa = rqctx.context(); let instance_id = path_params.into_inner().instance_id; let body_args = body.into_inner(); diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index baf92af28a..ce1ef662dc 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -6,9 +6,9 @@ use crate::common::instance::{ Action as InstanceAction, InstanceStates, ObservedPropolisState, - PublishedInstanceState, + PublishedVmmState, }; -use crate::instance_manager::InstanceTicket; +use crate::instance_manager::{InstanceManagerServices, InstanceTicket}; use crate::nexus::NexusClientWithResolver; use crate::params::ZoneBundleCause; use crate::params::ZoneBundleMetadata; @@ -22,6 +22,7 @@ use crate::zone_bundle::BundleError; use crate::zone_bundle::ZoneBundler; use anyhow::anyhow; use backoff::BackoffError; +use chrono::Utc; use futures::lock::{Mutex, MutexGuard}; use illumos_utils::dladm::Etherstub; use illumos_utils::link::VnicAllocator; @@ -32,7 +33,9 @@ use illumos_utils::zone::Zones; use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; use omicron_common::address::NEXUS_INTERNAL_PORT; use omicron_common::address::PROPOLIS_PORT; -use omicron_common::api::internal::nexus::InstanceRuntimeState; +use omicron_common::api::internal::nexus::{ + InstanceRuntimeState, SledInstanceState, VmmRuntimeState, +}; use omicron_common::api::internal::shared::{ NetworkInterface, SourceNatConfig, }; @@ -45,7 +48,6 @@ use slog::Logger; use std::net::IpAddr; use std::net::{SocketAddr, SocketAddrV6}; use std::sync::Arc; -use tokio::task::JoinHandle; use uuid::Uuid; #[derive(thiserror::Error, Debug)] @@ -171,34 +173,10 @@ enum Reaction { struct RunningState { // Connection to Propolis. client: Arc, - // Handle to task monitoring for Propolis state changes. - monitor_task: Option>, // Handle to the zone. running_zone: RunningZone, } -impl Drop for RunningState { - fn drop(&mut self) { - if let Some(task) = self.monitor_task.take() { - // NOTE: We'd prefer to actually await the task, since it - // will be completed at this point, but async drop doesn't exist. - // - // At a minimum, this implementation ensures the background task - // is not executing after RunningState terminates. - // - // "InstanceManager" contains... - // ... "Instance", which contains... - // ... "InstanceInner", which contains... - // ... "RunningState", which owns the "monitor_task". - // - // The "monitor_task" removes the instance from the - // "InstanceManager", triggering it's eventual drop. - // When this happens, the "monitor_task" exits anyway. - task.abort() - } - } -} - // Named type for values returned during propolis zone creation struct PropolisSetup { client: Arc, @@ -263,15 +241,15 @@ impl InstanceInner { &self.propolis_id } - async fn publish_state_to_nexus(&self) -> Result<(), Error> { + async fn publish_state_to_nexus(&self) { // Retry until Nexus acknowledges that it has applied this state update. // Note that Nexus may receive this call but then fail while reacting // to it. If that failure is transient, Nexus expects this routine to // retry the state update. - backoff::retry_notify( + let result = backoff::retry_notify( backoff::retry_policy_internal_service(), || async { - let state = self.state.current().clone(); + let state = self.state.sled_instance_state(); info!(self.log, "Publishing instance state update to Nexus"; "instance_id" => %self.id(), "state" => ?state, @@ -330,9 +308,15 @@ impl InstanceInner { "retry_after" => ?delay); }, ) - .await?; + .await; - Ok(()) + if let Err(e) = result { + error!( + self.log, + "Failed to publish state to Nexus, will not retry: {:?}", e; + "instance_id" => %self.id() + ); + } } /// Processes a Propolis state change observed by the Propolis monitoring @@ -365,28 +349,26 @@ impl InstanceInner { let action = self.state.apply_propolis_observation(state); info!( self.log, - "New state: {:?}, action: {:?}", - self.state.current().run_state, - action + "updated state after observing Propolis state change"; + "propolis_id" => %self.state.propolis_id(), + "new_instance_state" => ?self.state.instance(), + "new_vmm_state" => ?self.state.vmm() ); - // Publish the updated instance state to Nexus. The callee retries - // transient errors. If an error is permanent, log a message but - // continue monitoring so that the monitor will continue to take - // actions in response to future Propolis state changes. - if let Err(e) = self.publish_state_to_nexus().await { - let state = self.state.current(); - error!(self.log, - "Failed to publish state to Nexus, will not retry: {:?}", e; - "instance_id" => %self.id(), - "state" => ?state); - } + // If the zone is now safe to terminate, tear it down and discard the + // instance ticket before returning and publishing the new instance + // state to Nexus. This ensures that the instance is actually gone from + // the sled when Nexus receives the state update saying it's actually + // destroyed. + match action { + Some(InstanceAction::Destroy) => { + info!(self.log, "terminating VMM that has exited"; + "instance_id" => %self.id()); - // Take the next action, if any. - if let Some(action) = action { - self.take_action(action).await - } else { - Ok(Reaction::Continue) + self.terminate().await?; + Ok(Reaction::Terminate) + } + None => Ok(Reaction::Continue), } } @@ -434,7 +416,7 @@ impl InstanceInner { let migrate = match migrate { Some(params) => { let migration_id = - self.state.current().migration_id.ok_or_else(|| { + self.state.instance().migration_id.ok_or_else(|| { Error::Migration(anyhow!("Missing Migration UUID")) })?; Some(propolis_client::api::InstanceMigrateInitiateRequest { @@ -485,49 +467,26 @@ impl InstanceInner { self.propolis_ensure(&client, &running_zone, migrate).await?; // Monitor propolis for state changes in the background. + // + // This task exits after its associated Propolis has been terminated + // (either because the task observed a message from Propolis saying that + // it exited or because the Propolis server was terminated by other + // means). let monitor_client = client.clone(); - let monitor_task = Some(tokio::task::spawn(async move { + let _monitor_task = tokio::task::spawn(async move { let r = instance.monitor_state_task(monitor_client).await; let log = &instance.inner.lock().await.log; match r { Err(e) => warn!(log, "State monitoring task failed: {}", e), Ok(()) => info!(log, "State monitoring task complete"), } - })); + }); - self.running_state = - Some(RunningState { client, monitor_task, running_zone }); + self.running_state = Some(RunningState { client, running_zone }); Ok(()) } - async fn take_action( - &self, - action: InstanceAction, - ) -> Result { - info!(self.log, "Taking action: {:#?}", action); - let requested_state = match action { - InstanceAction::Run => { - propolis_client::api::InstanceStateRequested::Run - } - InstanceAction::Stop => { - propolis_client::api::InstanceStateRequested::Stop - } - InstanceAction::Reboot => { - propolis_client::api::InstanceStateRequested::Reboot - } - InstanceAction::Destroy => { - // Unlike the other actions, which update the Propolis state, - // the "destroy" action indicates that the service should be - // terminated. - info!(self.log, "take_action: Taking the Destroy action"); - return Ok(Reaction::Terminate); - } - }; - self.propolis_state_put(requested_state).await?; - Ok(Reaction::Continue) - } - /// Immediately terminates this instance's Propolis zone and cleans up any /// runtime objects associated with the instance. /// @@ -547,6 +506,10 @@ impl InstanceInner { self.log, "Instance::terminate() called with no running state" ); + + // Ensure the instance is removed from the instance manager's table + // so that a new instance can take its place. + self.instance_ticket.terminate(); return Ok(()); }; @@ -599,59 +562,84 @@ pub struct Instance { inner: Arc>, } +#[derive(Debug)] +pub(crate) struct InstanceInitialState { + pub hardware: InstanceHardware, + pub instance_runtime: InstanceRuntimeState, + pub vmm_runtime: VmmRuntimeState, + pub propolis_addr: SocketAddr, +} + impl Instance { /// Creates a new (not yet running) instance object. /// - /// Arguments: + /// # Arguments + /// /// * `log`: Logger for dumping debug information. /// * `id`: UUID of the instance to be created. - /// * `initial`: State of the instance at initialization time. - /// * `vnic_allocator`: A unique (to the sled) ID generator to - /// refer to a VNIC. (This exists because of a restriction on VNIC name - /// lengths, otherwise the UUID would be used instead). - /// * `port_manager`: Handle to the object responsible for managing OPTE - /// ports. - /// * `nexus_client`: Connection to Nexus, used for sending notifications. - // TODO: This arg list is getting a little long; can we clean this up? - #[allow(clippy::too_many_arguments)] - pub fn new( + /// * `propolis_id`: UUID for the VMM to be created. + /// * `ticket`: A ticket that ensures this instance is a member of its + /// instance manager's tracking table. + /// * `state`: The initial state of this instance. + /// * `services`: A set of instance manager-provided services. + pub(crate) fn new( log: Logger, id: Uuid, + propolis_id: Uuid, ticket: InstanceTicket, - initial: InstanceHardware, - vnic_allocator: VnicAllocator, - port_manager: PortManager, - nexus_client: NexusClientWithResolver, - storage: StorageResources, - zone_bundler: ZoneBundler, + state: InstanceInitialState, + services: InstanceManagerServices, ) -> Result { - info!(log, "Instance::new w/initial HW: {:?}", initial); + info!(log, "initializing new Instance"; + "instance_id" => %id, + "propolis_id" => %propolis_id, + "state" => ?state); + + let InstanceInitialState { + hardware, + instance_runtime, + vmm_runtime, + propolis_addr, + } = state; + + let InstanceManagerServices { + nexus_client, + vnic_allocator, + port_manager, + storage, + zone_bundler, + } = services; + let instance = InstanceInner { log: log.new(o!("instance_id" => id.to_string())), // NOTE: Mostly lies. properties: propolis_client::api::InstanceProperties { id, - name: initial.runtime.hostname.clone(), + name: hardware.properties.hostname.clone(), description: "Test description".to_string(), image_id: Uuid::nil(), bootrom_id: Uuid::nil(), // TODO: Align the byte type w/propolis. - memory: initial.runtime.memory.to_whole_mebibytes(), + memory: hardware.properties.memory.to_whole_mebibytes(), // TODO: we should probably make propolis aligned with // InstanceCpuCount here, to avoid any casting... - vcpus: initial.runtime.ncpus.0 as u8, + vcpus: hardware.properties.ncpus.0 as u8, }, - propolis_id: initial.runtime.propolis_id, - propolis_ip: initial.runtime.propolis_addr.unwrap().ip(), + propolis_id, + propolis_ip: propolis_addr.ip(), vnic_allocator, port_manager, - requested_nics: initial.nics, - source_nat: initial.source_nat, - external_ips: initial.external_ips, - firewall_rules: initial.firewall_rules, - requested_disks: initial.disks, - cloud_init_bytes: initial.cloud_init_bytes, - state: InstanceStates::new(initial.runtime), + requested_nics: hardware.nics, + source_nat: hardware.source_nat, + external_ips: hardware.external_ips, + firewall_rules: hardware.firewall_rules, + requested_disks: hardware.disks, + cloud_init_bytes: hardware.cloud_init_bytes, + state: InstanceStates::new( + instance_runtime, + vmm_runtime, + propolis_id, + ), running_state: None, nexus_client, storage, @@ -686,9 +674,9 @@ impl Instance { } } - pub async fn current_state(&self) -> InstanceRuntimeState { + pub async fn current_state(&self) -> SledInstanceState { let inner = self.inner.lock().await; - inner.state.current().clone() + inner.state.sled_instance_state() } /// Ensures that a Propolis process exists for this instance, then sends it @@ -712,25 +700,6 @@ impl Instance { .await?; } else { let setup_result: Result<(), Error> = 'setup: { - // If there's no Propolis yet, and this instance is not being - // initialized via migration, immediately send a state update to - // Nexus to reflect that the instance is starting (so that the - // external API will display this state while the zone is being - // started). - // - // Migration targets don't do this because the instance is still - // logically running (on the source) while the target Propolis - // is being launched. - if migration_params.is_none() { - info!(&inner.log, "Ensuring new instance"); - inner.state.transition(PublishedInstanceState::Starting); - if let Err(e) = inner.publish_state_to_nexus().await { - break 'setup Err(e); - } - } else { - info!(&inner.log, "Ensuring new instance (migration)"); - } - // Set up the Propolis zone and the objects associated with it. let setup = match self.setup_propolis_locked(inner).await { Ok(setup) => setup, @@ -757,9 +726,12 @@ impl Instance { // start a migration target simply leaves the VM running untouched // on the source. if migration_params.is_none() && setup_result.is_err() { - error!(&inner.log, "instance setup failed: {:?}", setup_result); - inner.state.transition(PublishedInstanceState::Failed); - inner.publish_state_to_nexus().await?; + error!(&inner.log, "vmm setup failed: {:?}", setup_result); + + // This case is morally equivalent to starting Propolis and then + // rudely terminating it before asking it to do anything. Update + // the VMM and instance states accordingly. + inner.state.terminate_rudely(); } setup_result?; } @@ -780,7 +752,7 @@ impl Instance { pub async fn put_state( &self, state: crate::params::InstanceStateRequested, - ) -> Result { + ) -> Result { use propolis_client::api::InstanceStateRequested as PropolisRequest; let mut inner = self.inner.lock().await; let (propolis_state, next_published) = match state { @@ -800,11 +772,12 @@ impl Instance { // "Destroyed" state and return it to the caller. if inner.running_state.is_none() { inner.terminate().await?; - (None, Some(PublishedInstanceState::Stopped)) + inner.state.terminate_rudely(); + (None, None) } else { ( Some(PropolisRequest::Stop), - Some(PublishedInstanceState::Stopping), + Some(PublishedVmmState::Stopping), ) } } @@ -814,7 +787,7 @@ impl Instance { } ( Some(PropolisRequest::Reboot), - Some(PublishedInstanceState::Rebooting), + Some(PublishedVmmState::Rebooting), ) } }; @@ -823,43 +796,43 @@ impl Instance { inner.propolis_state_put(p).await?; } if let Some(s) = next_published { - inner.state.transition(s); + inner.state.transition_vmm(s, Utc::now()); } - Ok(inner.state.current().clone()) + Ok(inner.state.sled_instance_state()) } pub async fn put_migration_ids( &self, old_runtime: &InstanceRuntimeState, migration_ids: &Option, - ) -> Result { + ) -> Result { let mut inner = self.inner.lock().await; // Check that the instance's current generation matches the one the // caller expects to transition from. This helps Nexus ensure that if // multiple migration sagas launch at Propolis generation N, then only // one of them will successfully set the instance's migration IDs. - if inner.state.current().propolis_gen != old_runtime.propolis_gen { + if inner.state.instance().gen != old_runtime.gen { // Allow this transition for idempotency if the instance is // already in the requested goal state. if inner.state.migration_ids_already_set(old_runtime, migration_ids) { - return Ok(inner.state.current().clone()); + return Ok(inner.state.sled_instance_state()); } return Err(Error::Transition( omicron_common::api::external::Error::Conflict { internal_message: format!( - "wrong Propolis ID generation: expected {}, got {}", - inner.state.current().propolis_gen, - old_runtime.propolis_gen + "wrong instance state generation: expected {}, got {}", + inner.state.instance().gen, + old_runtime.gen ), }, )); } - inner.state.set_migration_ids(migration_ids); - Ok(inner.state.current().clone()) + inner.state.set_migration_ids(migration_ids, Utc::now()); + Ok(inner.state.sled_instance_state()) } async fn setup_propolis_locked( @@ -979,7 +952,6 @@ impl Instance { info!(inner.log, "Propolis SMF service is online"); let server_addr = SocketAddr::new(inner.propolis_ip, PROPOLIS_PORT); - inner.state.current_mut().propolis_addr = Some(server_addr); // We use a custom client builder here because the default progenitor // one has a timeout of 15s but we want to be able to wait indefinitely. @@ -1000,11 +972,15 @@ impl Instance { /// Rudely terminates this instance's Propolis (if it has one) and /// immediately transitions the instance to the Destroyed state. - pub async fn terminate(&self) -> Result { + pub async fn terminate(&self) -> Result { let mut inner = self.inner.lock().await; inner.terminate().await?; - inner.state.transition(PublishedInstanceState::Stopped); - Ok(inner.state.current().clone()) + + // Rude termination is safe here because this routine took the lock + // before terminating the zone, which will cause any pending + // observations from the instance state monitor to be + inner.state.terminate_rudely(); + Ok(inner.state.sled_instance_state()) } // Monitors propolis until explicitly told to disconnect. @@ -1031,17 +1007,16 @@ impl Instance { // stabilize that state across this entire operation. let mut inner = self.inner.lock().await; let observed = ObservedPropolisState::new( - inner.state.current(), + inner.state.instance(), &response, ); - inner.observe_state(&observed).await? + let reaction = inner.observe_state(&observed).await?; + inner.publish_state_to_nexus().await; + reaction }; - match reaction { - Reaction::Continue => {} - Reaction::Terminate => { - return self.terminate().await.map(|_| ()); - } + if let Reaction::Terminate = reaction { + return Ok(()); } // Update the generation number we're asking for, to ensure the diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index bdd29e4d1f..2860f0624b 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -21,8 +21,11 @@ use illumos_utils::opte::PortManager; use illumos_utils::vmm_reservoir; use omicron_common::api::external::ByteCount; use omicron_common::api::internal::nexus::InstanceRuntimeState; +use omicron_common::api::internal::nexus::SledInstanceState; +use omicron_common::api::internal::nexus::VmmRuntimeState; use slog::Logger; use std::collections::BTreeMap; +use std::net::SocketAddr; use std::sync::{Arc, Mutex}; use uuid::Uuid; @@ -66,6 +69,14 @@ struct InstanceManagerInternal { zone_bundler: ZoneBundler, } +pub(crate) struct InstanceManagerServices { + pub nexus_client: NexusClientWithResolver, + pub vnic_allocator: VnicAllocator, + pub port_manager: PortManager, + pub storage: StorageResources, + pub zone_bundler: ZoneBundler, +} + /// All instances currently running on the sled. pub struct InstanceManager { inner: Arc, @@ -168,14 +179,21 @@ impl InstanceManager { pub async fn ensure_registered( &self, instance_id: Uuid, - initial_hardware: InstanceHardware, - ) -> Result { - let requested_propolis_id = initial_hardware.runtime.propolis_id; + propolis_id: Uuid, + hardware: InstanceHardware, + instance_runtime: InstanceRuntimeState, + vmm_runtime: VmmRuntimeState, + propolis_addr: SocketAddr, + ) -> Result { info!( &self.inner.log, "ensuring instance is registered"; "instance_id" => %instance_id, - "propolis_id" => %requested_propolis_id + "propolis_id" => %propolis_id, + "hardware" => ?hardware, + "instance_runtime" => ?instance_runtime, + "vmm_runtime" => ?vmm_runtime, + "propolis_addr" => ?propolis_addr, ); let instance = { @@ -183,7 +201,7 @@ impl InstanceManager { if let Some((existing_propolis_id, existing_instance)) = instances.get(&instance_id) { - if requested_propolis_id != *existing_propolis_id { + if propolis_id != *existing_propolis_id { info!(&self.inner.log, "instance already registered with another Propolis ID"; "instance_id" => %instance_id, @@ -207,20 +225,33 @@ impl InstanceManager { let instance_log = self.inner.log.new(o!()); let ticket = InstanceTicket::new(instance_id, self.inner.clone()); + + let services = InstanceManagerServices { + nexus_client: self.inner.nexus_client.clone(), + vnic_allocator: self.inner.vnic_allocator.clone(), + port_manager: self.inner.port_manager.clone(), + storage: self.inner.storage.clone(), + zone_bundler: self.inner.zone_bundler.clone(), + }; + + let state = crate::instance::InstanceInitialState { + hardware, + instance_runtime, + vmm_runtime, + propolis_addr, + }; + let instance = Instance::new( instance_log, instance_id, + propolis_id, ticket, - initial_hardware, - self.inner.vnic_allocator.clone(), - self.inner.port_manager.clone(), - self.inner.nexus_client.clone(), - self.inner.storage.clone(), - self.inner.zone_bundler.clone(), + state, + services, )?; let instance_clone = instance.clone(); - let _old = instances - .insert(instance_id, (requested_propolis_id, instance)); + let _old = + instances.insert(instance_id, (propolis_id, instance)); assert!(_old.is_none()); instance_clone } @@ -299,7 +330,7 @@ impl InstanceManager { instance_id: Uuid, old_runtime: &InstanceRuntimeState, migration_ids: &Option, - ) -> Result { + ) -> Result { let (_, instance) = self .inner .instances diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index d0fa2fbe4d..84ec1ef0dc 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -9,7 +9,8 @@ pub use crate::zone_bundle::ZoneBundleMetadata; pub use illumos_utils::opte::params::VpcFirewallRule; pub use illumos_utils::opte::params::VpcFirewallRulesEnsureBody; use omicron_common::api::internal::nexus::{ - DiskRuntimeState, InstanceRuntimeState, + DiskRuntimeState, InstanceProperties, InstanceRuntimeState, + SledInstanceState, VmmRuntimeState, }; use omicron_common::api::internal::shared::{ NetworkInterface, SourceNatConfig, @@ -60,7 +61,7 @@ pub struct DiskEnsureBody { /// Describes the instance hardware. #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] pub struct InstanceHardware { - pub runtime: InstanceRuntimeState, + pub properties: InstanceProperties, pub nics: Vec, pub source_nat: SourceNatConfig, /// Zero or more external IP addresses (either floating or ephemeral), @@ -72,12 +73,27 @@ pub struct InstanceHardware { pub cloud_init_bytes: Option, } -/// The body of a request to ensure that an instance is known to a sled agent. +/// The body of a request to ensure that a instance and VMM are known to a sled +/// agent. #[derive(Serialize, Deserialize, JsonSchema)] pub struct InstanceEnsureBody { /// A description of the instance's virtual hardware and the initial runtime /// state this sled agent should store for this incarnation of the instance. - pub initial: InstanceHardware, + pub hardware: InstanceHardware, + + /// The instance runtime state for the instance being registered. + pub instance_runtime: InstanceRuntimeState, + + /// The initial VMM runtime state for the VMM being registered. + pub vmm_runtime: VmmRuntimeState, + + /// The ID of the VMM being registered. This may not be the active VMM ID in + /// the instance runtime state (e.g. if the new VMM is going to be a + /// migration target). + pub propolis_id: Uuid, + + /// The address at which this VMM should serve a Propolis server API. + pub propolis_addr: SocketAddr, } /// The body of a request to move a previously-ensured instance into a specific @@ -95,7 +111,7 @@ pub struct InstancePutStateResponse { /// The current runtime state of the instance after handling the request to /// change its state. If the instance's state did not change, this field is /// `None`. - pub updated_runtime: Option, + pub updated_runtime: Option, } /// The response sent from a request to unregister an instance. @@ -104,7 +120,7 @@ pub struct InstanceUnregisterResponse { /// The current state of the instance after handling the request to /// unregister it. If the instance's state did not change, this field is /// `None`. - pub updated_runtime: Option, + pub updated_runtime: Option, } /// Parameters used when directing Propolis to initialize itself via live @@ -175,8 +191,8 @@ pub struct InstanceMigrationSourceParams { /// sled agent's instance state records. #[derive(Debug, Serialize, Deserialize, JsonSchema)] pub struct InstancePutMigrationIdsBody { - /// The last runtime state known to this requestor. This request will - /// succeed if either (a) the Propolis generation in the sled agent's + /// The last instance runtime state known to this requestor. This request + /// will succeed if either (a) the state generation in the sled agent's /// runtime state matches the generation in this record, or (b) the sled /// agent's runtime state matches what would result from applying this /// request to the caller's runtime state. This latter condition provides diff --git a/sled-agent/src/sim/collection.rs b/sled-agent/src/sim/collection.rs index ab6940b165..bd6ed4aa90 100644 --- a/sled-agent/src/sim/collection.rs +++ b/sled-agent/src/sim/collection.rs @@ -248,6 +248,9 @@ impl SimCollection { if object.object.desired().is_none() && object.object.ready_to_destroy() { + info!(&self.log, "object is ready to destroy"; + "object_id" => %id); + (after, Some(object)) } else { objects.insert(id, object); @@ -405,37 +408,42 @@ mod test { use chrono::Utc; use dropshot::test_util::LogContext; use futures::channel::mpsc::Receiver; - use omicron_common::api::external::ByteCount; use omicron_common::api::external::DiskState; use omicron_common::api::external::Error; use omicron_common::api::external::Generation; - use omicron_common::api::external::InstanceCpuCount; use omicron_common::api::external::InstanceState; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::InstanceRuntimeState; + use omicron_common::api::internal::nexus::SledInstanceState; + use omicron_common::api::internal::nexus::VmmRuntimeState; use omicron_test_utils::dev::test_setup_log; + use uuid::Uuid; fn make_instance( logctx: &LogContext, ) -> (SimObject, Receiver<()>) { - let initial_runtime = { - InstanceRuntimeState { - run_state: InstanceState::Creating, - sled_id: uuid::Uuid::new_v4(), - propolis_id: uuid::Uuid::new_v4(), - dst_propolis_id: None, - propolis_addr: None, - migration_id: None, - propolis_gen: Generation::new(), - ncpus: InstanceCpuCount(2), - memory: ByteCount::from_mebibytes_u32(512), - hostname: "myvm".to_string(), - gen: Generation::new(), - time_updated: Utc::now(), - } + let propolis_id = Uuid::new_v4(); + let instance_vmm = InstanceRuntimeState { + propolis_id: Some(propolis_id), + dst_propolis_id: None, + migration_id: None, + gen: Generation::new(), + time_updated: Utc::now(), }; - SimObject::new_simulated_auto(&initial_runtime, logctx.log.new(o!())) + let vmm_state = VmmRuntimeState { + state: InstanceState::Starting, + gen: Generation::new(), + time_updated: Utc::now(), + }; + + let state = SledInstanceState { + instance_state: instance_vmm, + vmm_state, + propolis_id, + }; + + SimObject::new_simulated_auto(&state, logctx.log.new(o!())) } fn make_disk( @@ -459,32 +467,38 @@ mod test { let (mut instance, mut rx) = make_instance(&logctx); let r1 = instance.object.current(); - info!(logctx.log, "new instance"; "run_state" => ?r1.run_state); - assert_eq!(r1.run_state, InstanceState::Creating); - assert_eq!(r1.gen, Generation::new()); + info!(logctx.log, "new instance"; "state" => ?r1); + assert_eq!(r1.vmm_state.state, InstanceState::Starting); + assert_eq!(r1.vmm_state.gen, Generation::new()); // There's no asynchronous transition going on yet so a // transition_finish() shouldn't change anything. assert!(instance.object.desired().is_none()); instance.transition_finish(); + let rnext = instance.object.current(); assert!(instance.object.desired().is_none()); - assert_eq!(&r1.time_updated, &instance.object.current().time_updated); - assert_eq!(&r1.run_state, &instance.object.current().run_state); - assert_eq!(r1.gen, instance.object.current().gen); + assert_eq!(r1.vmm_state.time_updated, rnext.vmm_state.time_updated); + assert_eq!(rnext.vmm_state.state, rnext.vmm_state.state); + assert_eq!(r1.vmm_state.gen, rnext.vmm_state.gen); assert!(rx.try_next().is_err()); - // Stopping an instance that was never started synchronously marks it - // stopped. + // Stopping an instance that was never started synchronously destroys + // its VMM. let rprev = r1; - assert!(rprev.run_state.is_stopped()); let dropped = instance.transition(InstanceStateRequested::Stopped).unwrap(); assert!(dropped.is_none()); assert!(instance.object.desired().is_none()); let rnext = instance.object.current(); - assert!(rnext.gen > rprev.gen); - assert!(rnext.time_updated >= rprev.time_updated); - assert_eq!(rnext.run_state, InstanceState::Stopped); + assert!(rnext.instance_state.gen > rprev.instance_state.gen); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); + assert!( + rnext.instance_state.time_updated + >= rprev.instance_state.time_updated + ); + assert!(rnext.vmm_state.time_updated >= rprev.vmm_state.time_updated); + assert!(rnext.instance_state.propolis_id.is_none()); + assert_eq!(rnext.vmm_state.state, InstanceState::Destroyed); assert!(rx.try_next().is_err()); logctx.cleanup_successful(); @@ -499,106 +513,115 @@ mod test { let (mut instance, mut rx) = make_instance(&logctx); let r1 = instance.object.current(); - info!(logctx.log, "new instance"; "run_state" => ?r1.run_state); - assert_eq!(r1.run_state, InstanceState::Creating); - assert_eq!(r1.gen, Generation::new()); + info!(logctx.log, "new instance"; "state" => ?r1); + assert_eq!(r1.vmm_state.state, InstanceState::Starting); + assert_eq!(r1.vmm_state.gen, Generation::new()); // There's no asynchronous transition going on yet so a // transition_finish() shouldn't change anything. assert!(instance.object.desired().is_none()); instance.transition_finish(); assert!(instance.object.desired().is_none()); - assert_eq!(&r1.time_updated, &instance.object.current().time_updated); - assert_eq!(&r1.run_state, &instance.object.current().run_state); - assert_eq!(r1.gen, instance.object.current().gen); + let rnext = instance.object.current(); + assert_eq!(r1.vmm_state.time_updated, rnext.vmm_state.time_updated); + assert_eq!(r1.vmm_state.state, rnext.vmm_state.state); + assert_eq!(r1.vmm_state.gen, rnext.vmm_state.gen); assert!(rx.try_next().is_err()); - // Now, if we transition to "Running", we must go through the async - // process. + // Set up a transition to Running. This has no immediate effect on the + // simulated instance's state, but it does queue up a transition. let mut rprev = r1; assert!(rx.try_next().is_err()); let dropped = instance.transition(InstanceStateRequested::Running).unwrap(); assert!(dropped.is_none()); assert!(instance.object.desired().is_some()); - assert!(rx.try_next().is_ok()); + assert!(rx.try_next().is_err()); + + // The VMM should still be Starting and its generation should not have + // changed (the transition to Running is queued but hasn't executed). let rnext = instance.object.current(); - assert!(rnext.gen > rprev.gen); - assert!(rnext.time_updated >= rprev.time_updated); - assert_eq!(rnext.run_state, InstanceState::Starting); - assert!(!rnext.run_state.is_stopped()); + assert_eq!(rnext.vmm_state.gen, rprev.vmm_state.gen); + assert_eq!(rnext.vmm_state.time_updated, rprev.vmm_state.time_updated); + assert_eq!(rnext.vmm_state.state, InstanceState::Starting); rprev = rnext; + // Now poke the instance. It should transition to Running. instance.transition_finish(); let rnext = instance.object.current(); - assert!(rnext.gen > rprev.gen); - assert!(rnext.time_updated >= rprev.time_updated); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); + assert!(rnext.vmm_state.time_updated >= rprev.vmm_state.time_updated); assert!(instance.object.desired().is_none()); assert!(rx.try_next().is_err()); - assert_eq!(rprev.run_state, InstanceState::Starting); - assert_eq!(rnext.run_state, InstanceState::Running); + assert_eq!(rprev.vmm_state.state, InstanceState::Starting); + assert_eq!(rnext.vmm_state.state, InstanceState::Running); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); rprev = rnext; + + // There shouldn't be anything left on the queue now. instance.transition_finish(); let rnext = instance.object.current(); - assert_eq!(rprev.gen, rnext.gen); + assert_eq!(rprev.vmm_state.gen, rnext.vmm_state.gen); // If we transition again to "Running", the process should complete // immediately. - assert!(!rprev.run_state.is_stopped()); let dropped = instance.transition(InstanceStateRequested::Running).unwrap(); assert!(dropped.is_none()); assert!(instance.object.desired().is_none()); assert!(rx.try_next().is_err()); let rnext = instance.object.current(); - assert_eq!(rnext.gen, rprev.gen); - assert_eq!(rnext.time_updated, rprev.time_updated); - assert_eq!(rnext.run_state, rprev.run_state); + assert_eq!(rnext.vmm_state.gen, rprev.vmm_state.gen); + assert_eq!(rnext.vmm_state.time_updated, rprev.vmm_state.time_updated); + assert_eq!(rnext.vmm_state.state, rprev.vmm_state.state); rprev = rnext; // If we go back to any stopped state, we go through the async process // again. - assert!(!rprev.run_state.is_stopped()); assert!(rx.try_next().is_err()); let dropped = instance.transition(InstanceStateRequested::Stopped).unwrap(); assert!(dropped.is_none()); assert!(instance.object.desired().is_some()); let rnext = instance.object.current(); - assert!(rnext.gen > rprev.gen); - assert!(rnext.time_updated >= rprev.time_updated); - assert_eq!(rnext.run_state, InstanceState::Stopping); - assert!(!rnext.run_state.is_stopped()); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); + assert!(rnext.vmm_state.time_updated >= rprev.vmm_state.time_updated); + assert_eq!(rnext.vmm_state.state, InstanceState::Stopping); rprev = rnext; // Propolis publishes its own transition to Stopping before it publishes // Stopped. instance.transition_finish(); let rnext = instance.object.current(); - assert!(rnext.gen > rprev.gen); - assert!(rnext.time_updated >= rprev.time_updated); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); + assert!(rnext.vmm_state.time_updated >= rprev.vmm_state.time_updated); assert!(instance.object.desired().is_some()); - assert_eq!(rprev.run_state, InstanceState::Stopping); - assert_eq!(rnext.run_state, InstanceState::Stopping); + assert_eq!(rprev.vmm_state.state, InstanceState::Stopping); + assert_eq!(rnext.vmm_state.state, InstanceState::Stopping); rprev = rnext; - // Stopping goes to Stopped... + // The Stopping-to-Stopped transition is masked from external viewers of + // the instance so that the instance doesn't appear to be Stopped before + // it is ready to be started again. instance.transition_finish(); let rnext = instance.object.current(); - assert!(rnext.gen > rprev.gen); - assert!(rnext.time_updated >= rprev.time_updated); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); + assert!(rnext.vmm_state.time_updated >= rprev.vmm_state.time_updated); assert!(instance.object.desired().is_some()); - assert_eq!(rprev.run_state, InstanceState::Stopping); - assert_eq!(rnext.run_state, InstanceState::Stopped); + assert_eq!(rprev.vmm_state.state, InstanceState::Stopping); + assert_eq!(rnext.vmm_state.state, InstanceState::Stopping); rprev = rnext; - // ...and Stopped (internally) goes to Destroyed, though the sled agent - // hides this state from clients. + // ...and Stopped (internally) goes to Destroyed. This transition is + // hidden from external viewers of the instance by retiring the active + // Propolis ID. instance.transition_finish(); let rnext = instance.object.current(); - assert!(rnext.gen > rprev.gen); - assert_eq!(rprev.run_state, InstanceState::Stopped); - assert_eq!(rnext.run_state, InstanceState::Stopped); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); + assert!(rnext.vmm_state.time_updated >= rprev.vmm_state.time_updated); + assert_eq!(rprev.vmm_state.state, InstanceState::Stopping); + assert_eq!(rnext.vmm_state.state, InstanceState::Destroyed); + assert!(rnext.instance_state.gen > rprev.instance_state.gen); logctx.cleanup_successful(); } @@ -611,9 +634,9 @@ mod test { let (mut instance, _rx) = make_instance(&logctx); let r1 = instance.object.current(); - info!(logctx.log, "new instance"; "run_state" => ?r1.run_state); - assert_eq!(r1.run_state, InstanceState::Creating); - assert_eq!(r1.gen, Generation::new()); + info!(logctx.log, "new instance"; "state" => ?r1); + assert_eq!(r1.vmm_state.state, InstanceState::Starting); + assert_eq!(r1.vmm_state.gen, Generation::new()); assert!(instance .transition(InstanceStateRequested::Running) .unwrap() @@ -626,7 +649,7 @@ mod test { std::thread::sleep(std::time::Duration::from_millis(100)); } - assert!(rnext.gen > rprev.gen); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); // Now reboot the instance. This is dispatched to Propolis, which will // move to the Rebooting state and then back to Running. @@ -635,9 +658,9 @@ mod test { .unwrap() .is_none()); let (rprev, rnext) = (rnext, instance.object.current()); - assert!(rnext.gen > rprev.gen); - assert!(rnext.time_updated > rprev.time_updated); - assert_eq!(rnext.run_state, InstanceState::Rebooting); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); + assert!(rnext.vmm_state.time_updated > rprev.vmm_state.time_updated); + assert_eq!(rnext.vmm_state.state, InstanceState::Rebooting); instance.transition_finish(); let (rprev, rnext) = (rnext, instance.object.current()); @@ -646,9 +669,9 @@ mod test { std::thread::sleep(std::time::Duration::from_millis(100)); } - assert!(rnext.gen > rprev.gen); - assert!(rnext.time_updated > rprev.time_updated); - assert_eq!(rnext.run_state, InstanceState::Rebooting); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); + assert!(rnext.vmm_state.time_updated > rprev.vmm_state.time_updated); + assert_eq!(rnext.vmm_state.state, InstanceState::Rebooting); assert!(instance.object.desired().is_some()); instance.transition_finish(); let (rprev, rnext) = (rnext, instance.object.current()); @@ -658,9 +681,9 @@ mod test { std::thread::sleep(std::time::Duration::from_millis(100)); } - assert!(rnext.gen > rprev.gen); - assert!(rnext.time_updated > rprev.time_updated); - assert_eq!(rnext.run_state, InstanceState::Running); + assert!(rnext.vmm_state.gen > rprev.vmm_state.gen); + assert!(rnext.vmm_state.time_updated > rprev.vmm_state.time_updated); + assert_eq!(rnext.vmm_state.state, InstanceState::Running); logctx.cleanup_successful(); } diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index bda34dec3f..08f6c7d10b 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -20,7 +20,7 @@ use dropshot::TypedBody; use illumos_utils::opte::params::DeleteVirtualNetworkInterfaceHost; use illumos_utils::opte::params::SetVirtualNetworkInterfaceHost; use omicron_common::api::internal::nexus::DiskRuntimeState; -use omicron_common::api::internal::nexus::InstanceRuntimeState; +use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::UpdateArtifactId; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -71,12 +71,19 @@ async fn instance_register( rqctx: RequestContext>, path_params: Path, body: TypedBody, -) -> Result, HttpError> { +) -> Result, HttpError> { let sa = rqctx.context(); let instance_id = path_params.into_inner().instance_id; let body_args = body.into_inner(); Ok(HttpResponseOk( - sa.instance_register(instance_id, body_args.initial).await?, + sa.instance_register( + instance_id, + body_args.propolis_id, + body_args.hardware, + body_args.instance_runtime, + body_args.vmm_runtime, + ) + .await?, )) } @@ -118,7 +125,7 @@ async fn instance_put_migration_ids( rqctx: RequestContext>, path_params: Path, body: TypedBody, -) -> Result, HttpError> { +) -> Result, HttpError> { let sa = rqctx.context(); let instance_id = path_params.into_inner().instance_id; let body_args = body.into_inner(); diff --git a/sled-agent/src/sim/instance.rs b/sled-agent/src/sim/instance.rs index 7283148563..397a1980a5 100644 --- a/sled-agent/src/sim/instance.rs +++ b/sled-agent/src/sim/instance.rs @@ -6,16 +6,19 @@ use super::simulatable::Simulatable; -use crate::common::instance::{ObservedPropolisState, PublishedInstanceState}; +use crate::common::instance::{ObservedPropolisState, PublishedVmmState}; use crate::nexus::NexusClient; use crate::params::{InstanceMigrationSourceParams, InstanceStateRequested}; use async_trait::async_trait; +use chrono::Utc; use nexus_client; use omicron_common::api::external::Error; use omicron_common::api::external::Generation; use omicron_common::api::external::InstanceState as ApiInstanceState; use omicron_common::api::external::ResourceType; -use omicron_common::api::internal::nexus::InstanceRuntimeState; +use omicron_common::api::internal::nexus::{ + InstanceRuntimeState, SledInstanceState, +}; use propolis_client::api::InstanceMigrateStatusResponse as PropolisMigrateStatus; use propolis_client::api::InstanceState as PropolisInstanceState; use propolis_client::api::InstanceStateMonitorResponse; @@ -96,54 +99,60 @@ impl SimInstanceInner { target: &InstanceStateRequested, ) -> Result, Error> { match target { + // When Nexus intends to migrate into a VMM, it should create that + // VMM in the Migrating state and shouldn't request anything else + // from it before asking to migrate in. InstanceStateRequested::MigrationTarget(_) => { - match self.next_resting_state() { - ApiInstanceState::Creating => { - self.queue_propolis_state( - PropolisInstanceState::Migrating, - ); - - let migration_id = - self.state.current().migration_id.expect( - "should have migration ID set before getting \ - request to migrate in", - ); - self.queue_migration_status(PropolisMigrateStatus { - migration_id, - state: propolis_client::api::MigrationState::Sync, - }); - self.queue_migration_status(PropolisMigrateStatus { - migration_id, - state: propolis_client::api::MigrationState::Finish, - }); - self.queue_propolis_state( - PropolisInstanceState::Running, - ); - } - _ => { - return Err(Error::invalid_request(&format!( - "can't request migration in with pending resting \ - state {}", - self.next_resting_state() - ))) - } + if !self.queue.is_empty() { + return Err(Error::invalid_request(&format!( + "can't request migration in with a non-empty state + transition queue (current state: {:?})", + self + ))); + } + if self.state.vmm().state != ApiInstanceState::Migrating { + return Err(Error::invalid_request(&format!( + "can't request migration in for a vmm that wasn't \ + created in the migrating state (current state: {:?})", + self + ))); } + + // Propolis transitions to the Migrating state once before + // actually starting migration. + self.queue_propolis_state(PropolisInstanceState::Migrating); + let migration_id = + self.state.instance().migration_id.unwrap_or_else(|| { + panic!( + "should have migration ID set before getting request to + migrate in (current state: {:?})", + self + ) + }); + self.queue_migration_status(PropolisMigrateStatus { + migration_id, + state: propolis_client::api::MigrationState::Sync, + }); + self.queue_migration_status(PropolisMigrateStatus { + migration_id, + state: propolis_client::api::MigrationState::Finish, + }); + self.queue_propolis_state(PropolisInstanceState::Running); } InstanceStateRequested::Running => { match self.next_resting_state() { - ApiInstanceState::Creating => { - // The non-simulated sled agent explicitly and - // synchronously publishes the "Starting" state when - // cold-booting a new VM (so that the VM appears to be - // starting while its Propolis process is being - // launched). - self.state.transition(PublishedInstanceState::Starting); + // It's only valid to request the Running state after + // successfully registering a VMM, and a registered VMM + // should never be in the Creating state. + ApiInstanceState::Creating => unreachable!( + "VMMs should never try to reach the Creating state" + ), + ApiInstanceState::Starting => { self.queue_propolis_state( PropolisInstanceState::Running, ); } - ApiInstanceState::Starting - | ApiInstanceState::Running + ApiInstanceState::Running | ApiInstanceState::Rebooting | ApiInstanceState::Migrating => {} @@ -157,19 +166,26 @@ impl SimInstanceInner { | ApiInstanceState::Destroyed => { return Err(Error::invalid_request(&format!( "can't request state Running with pending resting \ - state {}", - self.next_resting_state() + state {} (current state: {:?})", + self.next_resting_state(), + self ))) } } } InstanceStateRequested::Stopped => { match self.next_resting_state() { - ApiInstanceState::Creating => { - self.state.transition(PublishedInstanceState::Stopped); + ApiInstanceState::Creating => unreachable!( + "VMMs should never try to reach the Creating state" + ), + ApiInstanceState::Starting => { + self.state.terminate_rudely(); } ApiInstanceState::Running => { - self.state.transition(PublishedInstanceState::Stopping); + self.state.transition_vmm( + PublishedVmmState::Stopping, + Utc::now(), + ); self.queue_propolis_state( PropolisInstanceState::Stopping, ); @@ -188,8 +204,9 @@ impl SimInstanceInner { _ => { return Err(Error::invalid_request(&format!( "can't request state Stopped with pending resting \ - state {}", - self.next_resting_state() + state {} (current state: {:?})", + self.next_resting_state(), + self ))) } } @@ -198,12 +215,13 @@ impl SimInstanceInner { ApiInstanceState::Running => { // Further requests to reboot are ignored if the instance // is currently rebooting or about to reboot. - if self.state.current().run_state - != ApiInstanceState::Rebooting + if self.state.vmm().state != ApiInstanceState::Rebooting && !self.reboot_pending() { - self.state - .transition(PublishedInstanceState::Rebooting); + self.state.transition_vmm( + PublishedVmmState::Rebooting, + Utc::now(), + ); self.queue_propolis_state( PropolisInstanceState::Rebooting, ); @@ -214,8 +232,10 @@ impl SimInstanceInner { } _ => { return Err(Error::invalid_request(&format!( - "can't request Reboot with pending resting state {}", - self.next_resting_state() + "can't request Reboot with pending resting state {} \ + (current state: {:?})", + self.next_resting_state(), + self ))) } }, @@ -240,7 +260,7 @@ impl SimInstanceInner { } self.state.apply_propolis_observation(&ObservedPropolisState::new( - &self.current(), + &self.state.instance(), &self.last_response, )) } else { @@ -248,11 +268,6 @@ impl SimInstanceInner { } } - /// Yields the current simulated instance runtime state. - fn current(&self) -> InstanceRuntimeState { - self.state.current().clone() - } - /// If the state change queue contains at least once instance state change, /// returns the requested instance state associated with the last instance /// state on the queue. Returns None otherwise. @@ -291,15 +306,26 @@ impl SimInstanceInner { /// queue is drained. fn next_resting_state(&self) -> ApiInstanceState { if self.queue.is_empty() { - self.state.current().run_state + self.state.vmm().state } else { if let Some(last_state) = self.last_queued_instance_state() { - crate::common::instance::PublishedInstanceState::from( - last_state, - ) - .into() + use ApiInstanceState as ApiState; + use PropolisInstanceState as PropolisState; + match last_state { + PropolisState::Creating | PropolisState::Starting => { + ApiState::Starting + } + PropolisState::Running => ApiState::Running, + PropolisState::Stopping => ApiState::Stopping, + PropolisState::Stopped => ApiState::Stopped, + PropolisState::Rebooting => ApiState::Rebooting, + PropolisState::Migrating => ApiState::Migrating, + PropolisState::Repairing => ApiState::Repairing, + PropolisState::Failed => ApiState::Failed, + PropolisState::Destroyed => ApiState::Destroyed, + } } else { - self.state.current().run_state + self.state.vmm().state } } } @@ -317,10 +343,11 @@ impl SimInstanceInner { /// Simulates rude termination by moving the instance to the Destroyed state /// immediately and clearing the queue of pending state transitions. - fn terminate(&mut self) -> InstanceRuntimeState { - self.state.transition(PublishedInstanceState::Stopped); + fn terminate(&mut self) -> SledInstanceState { + self.state.terminate_rudely(); self.queue.clear(); - self.state.current().clone() + self.destroyed = true; + self.state.sled_instance_state() } /// Stores a set of migration IDs in the instance's runtime state. @@ -328,23 +355,23 @@ impl SimInstanceInner { &mut self, old_runtime: &InstanceRuntimeState, ids: &Option, - ) -> Result { + ) -> Result { if self.state.migration_ids_already_set(old_runtime, ids) { - return Ok(self.state.current().clone()); + return Ok(self.state.sled_instance_state()); } - if self.state.current().propolis_gen != old_runtime.propolis_gen { + if self.state.instance().gen != old_runtime.gen { return Err(Error::InvalidRequest { message: format!( "wrong Propolis ID generation: expected {}, got {}", - self.state.current().propolis_gen, - old_runtime.propolis_gen + self.state.instance().gen, + old_runtime.gen ), }); } - self.state.set_migration_ids(ids); - Ok(self.state.current().clone()) + self.state.set_migration_ids(ids, Utc::now()); + Ok(self.state.sled_instance_state()) } } @@ -369,7 +396,7 @@ pub struct SimInstance { } impl SimInstance { - pub fn terminate(&self) -> InstanceRuntimeState { + pub fn terminate(&self) -> SledInstanceState { self.inner.lock().unwrap().terminate() } @@ -377,7 +404,7 @@ impl SimInstance { &self, old_runtime: &InstanceRuntimeState, ids: &Option, - ) -> Result { + ) -> Result { let mut inner = self.inner.lock().unwrap(); inner.put_migration_ids(old_runtime, ids) } @@ -385,18 +412,30 @@ impl SimInstance { #[async_trait] impl Simulatable for SimInstance { - type CurrentState = InstanceRuntimeState; + type CurrentState = SledInstanceState; type RequestedState = InstanceStateRequested; type ProducerArgs = (); type Action = InstanceAction; - fn new(current: InstanceRuntimeState) -> Self { + fn new(current: SledInstanceState) -> Self { + assert!(matches!( + current.vmm_state.state, + ApiInstanceState::Starting | ApiInstanceState::Migrating), + "new VMMs should always be registered in the Starting or Migrating \ + state (supplied state: {:?})", + current.vmm_state.state + ); + SimInstance { inner: Arc::new(Mutex::new(SimInstanceInner { - state: InstanceStates::new(current), + state: InstanceStates::new( + current.instance_state, + current.vmm_state, + current.propolis_id, + ), last_response: InstanceStateMonitorResponse { gen: 1, - state: PropolisInstanceState::Creating, + state: PropolisInstanceState::Starting, migration: None, }, queue: VecDeque::new(), @@ -425,11 +464,11 @@ impl Simulatable for SimInstance { } fn generation(&self) -> Generation { - self.inner.lock().unwrap().current().gen + self.inner.lock().unwrap().state.vmm().gen } fn current(&self) -> Self::CurrentState { - self.inner.lock().unwrap().current() + self.inner.lock().unwrap().state.sled_instance_state() } fn desired(&self) -> Option { @@ -448,7 +487,7 @@ impl Simulatable for SimInstance { nexus_client .cpapi_instances_put( id, - &nexus_client::types::InstanceRuntimeState::from(current), + &nexus_client::types::SledInstanceState::from(current), ) .await .map(|_| ()) diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 42fff355a5..e4dac2f4b9 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -21,8 +21,12 @@ use crate::sim::simulatable::Simulatable; use crate::updates::UpdateManager; use futures::lock::Mutex; use omicron_common::api::external::{DiskState, Error, ResourceType}; -use omicron_common::api::internal::nexus::DiskRuntimeState; -use omicron_common::api::internal::nexus::InstanceRuntimeState; +use omicron_common::api::internal::nexus::{ + DiskRuntimeState, SledInstanceState, +}; +use omicron_common::api::internal::nexus::{ + InstanceRuntimeState, VmmRuntimeState, +}; use slog::Logger; use std::net::{IpAddr, Ipv6Addr, SocketAddr}; use std::sync::Arc; @@ -219,18 +223,21 @@ impl SledAgent { pub async fn instance_register( self: &Arc, instance_id: Uuid, - mut initial_hardware: InstanceHardware, - ) -> Result { + propolis_id: Uuid, + hardware: InstanceHardware, + instance_runtime: InstanceRuntimeState, + vmm_runtime: VmmRuntimeState, + ) -> Result { // respond with a fake 500 level failure if asked to ensure an instance // with more than 16 CPUs. - let ncpus: i64 = (&initial_hardware.runtime.ncpus).into(); + let ncpus: i64 = (&hardware.properties.ncpus).into(); if ncpus > 16 { return Err(Error::internal_error( &"could not allocate an instance: ran out of CPUs!", )); }; - for disk in &initial_hardware.disks { + for disk in &hardware.disks { let initial_state = DiskRuntimeState { disk_state: DiskState::Attached(instance_id), gen: omicron_common::api::external::Generation::new(), @@ -255,27 +262,24 @@ impl SledAgent { .await?; } - // if we're making our first instance and a mock propolis-server - // is running, interact with it, and patch the instance's - // reported propolis-server IP for reports back to nexus. + // If the user of this simulated agent previously requested a mock + // Propolis server, start that server. + // + // N.B. The server serves on localhost and not on the per-sled IPv6 + // address that Nexus chose when starting the instance. Tests that + // use the mock are expected to correct the contents of CRDB to + // point to the correct address. let mock_lock = self.mock_propolis.lock().await; if let Some((_srv, client)) = mock_lock.as_ref() { - if let Some(addr) = initial_hardware.runtime.propolis_addr.as_mut() - { - addr.set_ip(Ipv6Addr::LOCALHOST.into()); - } if !self.instances.contains_key(&instance_id).await { let properties = propolis_client::types::InstanceProperties { - id: initial_hardware.runtime.propolis_id, - name: initial_hardware.runtime.hostname.clone(), + id: propolis_id, + name: hardware.properties.hostname.clone(), description: "sled-agent-sim created instance".to_string(), image_id: Uuid::default(), bootrom_id: Uuid::default(), - memory: initial_hardware - .runtime - .memory - .to_whole_mebibytes(), - vcpus: initial_hardware.runtime.ncpus.0 as u8, + memory: hardware.properties.memory.to_whole_mebibytes(), + vcpus: hardware.properties.ncpus.0 as u8, }; let body = propolis_client::types::InstanceEnsureRequest { properties, @@ -298,10 +302,18 @@ impl SledAgent { let instance_run_time_state = self .instances - .sim_ensure(&instance_id, initial_hardware.runtime, None) + .sim_ensure( + &instance_id, + SledInstanceState { + instance_state: instance_runtime, + vmm_state: vmm_runtime, + propolis_id, + }, + None, + ) .await?; - for disk_request in &initial_hardware.disks { + for disk_request in &hardware.disks { let vcr = &disk_request.volume_construction_request; self.map_disk_ids_to_region_ids(&vcr).await?; } @@ -328,9 +340,20 @@ impl SledAgent { }; self.detach_disks_from_instance(instance_id).await?; - Ok(InstanceUnregisterResponse { + let response = InstanceUnregisterResponse { updated_runtime: Some(instance.terminate()), - }) + }; + + // Poke the now-destroyed instance to force it to be removed from the + // collection. + // + // TODO: In the real sled agent, this happens inline without publishing + // any other state changes, whereas this call causes any pending state + // changes to be published. This can be fixed by adding a simulated + // object collection function to forcibly remove an object from a + // collection. + self.instances.sim_poke(instance_id, PokeMode::Drain).await; + Ok(response) } /// Asks the supplied instance to transition to the requested state. @@ -418,7 +441,7 @@ impl SledAgent { instance_id: Uuid, old_runtime: &InstanceRuntimeState, migration_ids: &Option, - ) -> Result { + ) -> Result { let instance = self.instances.sim_get_cloned_object(&instance_id).await?; diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 5574edca55..b6f910220e 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -35,6 +35,9 @@ use omicron_common::address::{ get_sled_address, get_switch_zone_address, Ipv6Subnet, SLED_PREFIX, }; use omicron_common::api::external::Vni; +use omicron_common::api::internal::nexus::{ + SledInstanceState, VmmRuntimeState, +}; use omicron_common::api::internal::shared::RackNetworkConfig; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, @@ -48,7 +51,7 @@ use sled_hardware::underlay; use sled_hardware::HardwareManager; use slog::Logger; use std::collections::BTreeMap; -use std::net::{Ipv6Addr, SocketAddrV6}; +use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -770,11 +773,22 @@ impl SledAgent { pub async fn instance_ensure_registered( &self, instance_id: Uuid, - initial: InstanceHardware, - ) -> Result { + propolis_id: Uuid, + hardware: InstanceHardware, + instance_runtime: InstanceRuntimeState, + vmm_runtime: VmmRuntimeState, + propolis_addr: SocketAddr, + ) -> Result { self.inner .instances - .ensure_registered(instance_id, initial) + .ensure_registered( + instance_id, + propolis_id, + hardware, + instance_runtime, + vmm_runtime, + propolis_addr, + ) .await .map_err(|e| Error::Instance(e)) } @@ -818,7 +832,7 @@ impl SledAgent { instance_id: Uuid, old_runtime: &InstanceRuntimeState, migration_ids: &Option, - ) -> Result { + ) -> Result { self.inner .instances .put_migration_ids(instance_id, old_runtime, migration_ids) From 7e88bdffbae7a8796a4ebc6a5bfe80d1fedd4bb7 Mon Sep 17 00:00:00 2001 From: Greg Colombo Date: Thu, 12 Oct 2023 11:06:18 -0700 Subject: [PATCH 48/89] Move instance/VMM table schema upgrade to version 7.0.0 (#4270) The instance/VMM table schema change was slated to be 6.0.0 in the original version of its pull request. That version was then added by a separate PR, but this didn't cause a merge conflict because the instance/VMM upgrade used an extra "0" in its schema upgrade files (for fear that there might be more than nine of them and that the leading 0 would be necessary to ensure they had the correct lexographical ordering). The schema changes don't conflict with each other, so everything (probably) works fine, but having two logically separate updates in one version is at the very least aesthetically displeasing. Move the instance schema upgrade to version 7.0.0. Rename the files to remove the leading 0 in their numbers, since that turned out not to be needed. Tested via cargo tests (there are no other functional or schema changes beyond renaming and updating version constants). --- dev-tools/omdb/tests/env.out | 6 +++--- dev-tools/omdb/tests/successes.out | 12 ++++++------ nexus/db-model/src/schema.rs | 2 +- schema/crdb/{6.0.0 => 7.0.0}/README.adoc | 0 schema/crdb/{6.0.0/up01.sql => 7.0.0/up1.sql} | 0 schema/crdb/{6.0.0/up02.sql => 7.0.0/up2.sql} | 0 schema/crdb/{6.0.0/up03.sql => 7.0.0/up3.sql} | 0 schema/crdb/{6.0.0/up04.sql => 7.0.0/up4.sql} | 0 schema/crdb/{6.0.0/up05.sql => 7.0.0/up5.sql} | 0 schema/crdb/{6.0.0/up06.sql => 7.0.0/up6.sql} | 0 schema/crdb/{6.0.0/up07.sql => 7.0.0/up7.sql} | 0 schema/crdb/{6.0.0/up08.sql => 7.0.0/up8.sql} | 0 schema/crdb/{6.0.0/up09.sql => 7.0.0/up9.sql} | 0 schema/crdb/dbinit.sql | 2 +- 14 files changed, 11 insertions(+), 11 deletions(-) rename schema/crdb/{6.0.0 => 7.0.0}/README.adoc (100%) rename schema/crdb/{6.0.0/up01.sql => 7.0.0/up1.sql} (100%) rename schema/crdb/{6.0.0/up02.sql => 7.0.0/up2.sql} (100%) rename schema/crdb/{6.0.0/up03.sql => 7.0.0/up3.sql} (100%) rename schema/crdb/{6.0.0/up04.sql => 7.0.0/up4.sql} (100%) rename schema/crdb/{6.0.0/up05.sql => 7.0.0/up5.sql} (100%) rename schema/crdb/{6.0.0/up06.sql => 7.0.0/up6.sql} (100%) rename schema/crdb/{6.0.0/up07.sql => 7.0.0/up7.sql} (100%) rename schema/crdb/{6.0.0/up08.sql => 7.0.0/up8.sql} (100%) rename schema/crdb/{6.0.0/up09.sql => 7.0.0/up9.sql} (100%) diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 07a6d3fae5..8e345b78d1 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -7,7 +7,7 @@ sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (6.0.0) +note: database schema version matches expected (7.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "--db-url", "junk", "sleds"] termination: Exited(2) @@ -172,7 +172,7 @@ stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (6.0.0) +note: database schema version matches expected (7.0.0) ============================================= EXECUTING COMMAND: omdb ["--dns-server", "[::1]:REDACTED_PORT", "db", "sleds"] termination: Exited(0) @@ -185,5 +185,5 @@ stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (6.0.0) +note: database schema version matches expected (7.0.0) ============================================= diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 038f365e8e..6fd84c5eb3 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -8,7 +8,7 @@ external oxide-dev.test 2 create silo: "tes --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (6.0.0) +note: database schema version matches expected (7.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "dns", "diff", "external", "2"] termination: Exited(0) @@ -24,7 +24,7 @@ changes: names added: 1, names removed: 0 --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (6.0.0) +note: database schema version matches expected (7.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "dns", "names", "external", "2"] termination: Exited(0) @@ -36,7 +36,7 @@ External zone: oxide-dev.test --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (6.0.0) +note: database schema version matches expected (7.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-instances"] termination: Exited(0) @@ -52,7 +52,7 @@ Nexus REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_ --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (6.0.0) +note: database schema version matches expected (7.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-by-sled"] termination: Exited(0) @@ -71,7 +71,7 @@ sled: sim-b6d65341 (id REDACTED_UUID_REDACTED_UUID_REDACTED) --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (6.0.0) +note: database schema version matches expected (7.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "sleds"] termination: Exited(0) @@ -82,7 +82,7 @@ sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (6.0.0) +note: database schema version matches expected (7.0.0) ============================================= EXECUTING COMMAND: omdb ["mgs", "inventory"] termination: Exited(0) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 2d6970452d..61a05754c6 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1142,7 +1142,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(6, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(7, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/schema/crdb/6.0.0/README.adoc b/schema/crdb/7.0.0/README.adoc similarity index 100% rename from schema/crdb/6.0.0/README.adoc rename to schema/crdb/7.0.0/README.adoc diff --git a/schema/crdb/6.0.0/up01.sql b/schema/crdb/7.0.0/up1.sql similarity index 100% rename from schema/crdb/6.0.0/up01.sql rename to schema/crdb/7.0.0/up1.sql diff --git a/schema/crdb/6.0.0/up02.sql b/schema/crdb/7.0.0/up2.sql similarity index 100% rename from schema/crdb/6.0.0/up02.sql rename to schema/crdb/7.0.0/up2.sql diff --git a/schema/crdb/6.0.0/up03.sql b/schema/crdb/7.0.0/up3.sql similarity index 100% rename from schema/crdb/6.0.0/up03.sql rename to schema/crdb/7.0.0/up3.sql diff --git a/schema/crdb/6.0.0/up04.sql b/schema/crdb/7.0.0/up4.sql similarity index 100% rename from schema/crdb/6.0.0/up04.sql rename to schema/crdb/7.0.0/up4.sql diff --git a/schema/crdb/6.0.0/up05.sql b/schema/crdb/7.0.0/up5.sql similarity index 100% rename from schema/crdb/6.0.0/up05.sql rename to schema/crdb/7.0.0/up5.sql diff --git a/schema/crdb/6.0.0/up06.sql b/schema/crdb/7.0.0/up6.sql similarity index 100% rename from schema/crdb/6.0.0/up06.sql rename to schema/crdb/7.0.0/up6.sql diff --git a/schema/crdb/6.0.0/up07.sql b/schema/crdb/7.0.0/up7.sql similarity index 100% rename from schema/crdb/6.0.0/up07.sql rename to schema/crdb/7.0.0/up7.sql diff --git a/schema/crdb/6.0.0/up08.sql b/schema/crdb/7.0.0/up8.sql similarity index 100% rename from schema/crdb/6.0.0/up08.sql rename to schema/crdb/7.0.0/up8.sql diff --git a/schema/crdb/6.0.0/up09.sql b/schema/crdb/7.0.0/up9.sql similarity index 100% rename from schema/crdb/6.0.0/up09.sql rename to schema/crdb/7.0.0/up9.sql diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 2b06e4cbd6..9f5f78326c 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -2539,7 +2539,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '6.0.0', NULL) + ( TRUE, NOW(), NOW(), '7.0.0', NULL) ON CONFLICT DO NOTHING; From 7d5538267e45737d951df440879d96cea533592f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 12 Oct 2023 14:54:57 -0700 Subject: [PATCH 49/89] [oximeter] Use stable hash, stable format, for deriving timeseries_key (#4251) - Uses [bcs](https://crates.io/crates/bcs) as a stable binary format for inputs to the `timeseries_key` function - Uses [highway](https://crates.io/crates/highway) as a stable hash algorithm (it's keyed, fast, portable, and well-distributed). - Additionally, adds an EXPECTORATE test validating the stability of timeseries_key values. Fixes https://github.com/oxidecomputer/omicron/issues/4008 , and also addresses the issue raised in https://github.com/oxidecomputer/omicron/issues/4221 regarding stable input NOTE: This PR itself *also* breaks the stability of the `timeseries_key` (hopefully for the last time), and will rely on https://github.com/oxidecomputer/omicron/pull/4246 to wipe the metrics DB for the next release. --- Cargo.lock | 21 +++ Cargo.toml | 1 + oximeter/db/Cargo.toml | 4 + oximeter/db/src/lib.rs | 130 ++++++++++++++++-- .../db/test-output/field-timeseries-keys.txt | 12 ++ .../db/test-output/sample-timeseries-key.txt | 1 + oximeter/oximeter/Cargo.toml | 1 + oximeter/oximeter/src/types.rs | 20 ++- 8 files changed, 180 insertions(+), 10 deletions(-) create mode 100644 oximeter/db/test-output/field-timeseries-keys.txt create mode 100644 oximeter/db/test-output/sample-timeseries-key.txt diff --git a/Cargo.lock b/Cargo.lock index d5a90f7f85..ce17dbe311 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -493,6 +493,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "bcs" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd3ffe8b19a604421a5d461d4a70346223e535903fbc3067138bddbebddcf77" +dependencies = [ + "serde", + "thiserror", +] + [[package]] name = "bhyve_api" version = "0.0.0" @@ -3098,6 +3108,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" +[[package]] +name = "highway" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ba82c000837f4e74df01a5520f0dc48735d4aed955a99eae4428bab7cf3acd" + [[package]] name = "hkdf" version = "0.12.3" @@ -5732,6 +5748,7 @@ dependencies = [ "rstest", "schemars", "serde", + "strum", "thiserror", "trybuild", "uuid", @@ -5810,10 +5827,13 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "bcs", "bytes", "chrono", "clap 4.4.3", "dropshot", + "expectorate", + "highway", "itertools 0.11.0", "omicron-test-utils", "omicron-workspace-hack", @@ -5827,6 +5847,7 @@ dependencies = [ "slog-async", "slog-dtrace", "slog-term", + "strum", "thiserror", "tokio", "usdt", diff --git a/Cargo.toml b/Cargo.toml index 7521bb4d45..5c10a94706 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -199,6 +199,7 @@ headers = "0.3.9" heck = "0.4" hex = "0.4.3" hex-literal = "0.4.1" +highway = "1.1.0" hkdf = "0.12.3" http = "0.2.9" httptest = "0.15.4" diff --git a/oximeter/db/Cargo.toml b/oximeter/db/Cargo.toml index ad6d584b1b..d37c57ccce 100644 --- a/oximeter/db/Cargo.toml +++ b/oximeter/db/Cargo.toml @@ -8,10 +8,12 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true async-trait.workspace = true +bcs.workspace = true bytes = { workspace = true, features = [ "serde" ] } chrono.workspace = true clap.workspace = true dropshot.workspace = true +highway.workspace = true oximeter.workspace = true regex.workspace = true reqwest = { workspace = true, features = [ "json" ] } @@ -28,9 +30,11 @@ uuid.workspace = true omicron-workspace-hack.workspace = true [dev-dependencies] +expectorate.workspace = true itertools.workspace = true omicron-test-utils.workspace = true slog-dtrace.workspace = true +strum.workspace = true [[bin]] name = "oxdb" diff --git a/oximeter/db/src/lib.rs b/oximeter/db/src/lib.rs index 9720d6914d..c878b8ff2a 100644 --- a/oximeter/db/src/lib.rs +++ b/oximeter/db/src/lib.rs @@ -328,19 +328,46 @@ pub struct TimeseriesPageSelector { pub(crate) type TimeseriesKey = u64; pub(crate) fn timeseries_key(sample: &Sample) -> TimeseriesKey { - timeseries_key_for(sample.target_fields(), sample.metric_fields()) + timeseries_key_for( + &sample.timeseries_name, + sample.sorted_target_fields(), + sample.sorted_metric_fields(), + sample.measurement.datum_type(), + ) } -pub(crate) fn timeseries_key_for<'a>( - target_fields: impl Iterator, - metric_fields: impl Iterator, +// It's critical that values used for derivation of the timeseries_key are stable. +// We use "bcs" to ensure stability of the derivation across hardware and rust toolchain revisions. +fn canonicalize(what: &str, value: &T) -> Vec { + bcs::to_bytes(value) + .unwrap_or_else(|_| panic!("Failed to serialize {what}")) +} + +fn timeseries_key_for( + timeseries_name: &str, + target_fields: &BTreeMap, + metric_fields: &BTreeMap, + datum_type: DatumType, ) -> TimeseriesKey { - use std::collections::hash_map::DefaultHasher; + // We use HighwayHasher primarily for stability - it should provide a stable + // hash for the values used to derive the timeseries_key. + use highway::HighwayHasher; use std::hash::{Hash, Hasher}; - let mut hasher = DefaultHasher::new(); - for field in target_fields.chain(metric_fields) { - field.hash(&mut hasher); + + // NOTE: The order of these ".hash" calls matters, changing them will change + // the derivation of the "timeseries_key". We have change-detector tests for + // modifications like this, but be cautious, making such a change will + // impact all currently-provisioned databases. + let mut hasher = HighwayHasher::default(); + canonicalize("timeseries name", timeseries_name).hash(&mut hasher); + for field in target_fields.values() { + canonicalize("target field", &field).hash(&mut hasher); } + for field in metric_fields.values() { + canonicalize("metric field", &field).hash(&mut hasher); + } + canonicalize("datum type", &datum_type).hash(&mut hasher); + hasher.finish() } @@ -370,8 +397,9 @@ const TIMESERIES_NAME_REGEX: &str = #[cfg(test)] mod tests { - use super::TimeseriesName; + use super::*; use std::convert::TryFrom; + use uuid::Uuid; #[test] fn test_timeseries_name() { @@ -393,4 +421,88 @@ mod tests { assert!(TimeseriesName::try_from("a:").is_err()); assert!(TimeseriesName::try_from("123").is_err()); } + + // Validates that the timeseries_key stability for a sample is stable. + #[test] + fn test_timeseries_key_sample_stability() { + #[derive(oximeter::Target)] + pub struct TestTarget { + pub name: String, + pub num: i64, + } + + #[derive(oximeter::Metric)] + pub struct TestMetric { + pub id: Uuid, + pub datum: i64, + } + + let target = TestTarget { name: String::from("Hello"), num: 1337 }; + let metric = TestMetric { id: Uuid::nil(), datum: 0x1de }; + let sample = Sample::new(&target, &metric).unwrap(); + let key = super::timeseries_key(&sample); + + expectorate::assert_contents( + "test-output/sample-timeseries-key.txt", + &format!("{key}"), + ); + } + + // Validates that the timeseries_key stability for specific fields is + // stable. + #[test] + fn test_timeseries_key_field_stability() { + use oximeter::{Field, FieldValue}; + use strum::EnumCount; + + let values = [ + ("string", FieldValue::String(String::default())), + ("i8", FieldValue::I8(-0x0A)), + ("u8", FieldValue::U8(0x0A)), + ("i16", FieldValue::I16(-0x0ABC)), + ("u16", FieldValue::U16(0x0ABC)), + ("i32", FieldValue::I32(-0x0ABC_0000)), + ("u32", FieldValue::U32(0x0ABC_0000)), + ("i64", FieldValue::I64(-0x0ABC_0000_0000_0000)), + ("u64", FieldValue::U64(0x0ABC_0000_0000_0000)), + ( + "ipaddr", + FieldValue::IpAddr(std::net::IpAddr::V4( + std::net::Ipv4Addr::LOCALHOST, + )), + ), + ("uuid", FieldValue::Uuid(uuid::Uuid::nil())), + ("bool", FieldValue::Bool(true)), + ]; + + // Exhaustively testing enums is a bit tricky. Although it's easy to + // check "all variants of an enum are matched", it harder to test "all + // variants of an enum have been supplied". + // + // We use this as a proxy, confirming that each variant is represented + // here for the purposes of tracking stability. + assert_eq!(values.len(), FieldValue::COUNT); + + let mut output = vec![]; + for (name, value) in values { + let target_fields = BTreeMap::from([( + "field".to_string(), + Field { name: name.to_string(), value }, + )]); + let metric_fields = BTreeMap::new(); + let key = timeseries_key_for( + "timeseries name", + &target_fields, + &metric_fields, + // ... Not actually, but we are only trying to compare fields here. + DatumType::Bool, + ); + output.push(format!("{name} -> {key}")); + } + + expectorate::assert_contents( + "test-output/field-timeseries-keys.txt", + &output.join("\n"), + ); + } } diff --git a/oximeter/db/test-output/field-timeseries-keys.txt b/oximeter/db/test-output/field-timeseries-keys.txt new file mode 100644 index 0000000000..d82c143600 --- /dev/null +++ b/oximeter/db/test-output/field-timeseries-keys.txt @@ -0,0 +1,12 @@ +string -> 5554437373902071418 +i8 -> 8051527130089763326 +u8 -> 1403385090410880239 +i16 -> 4425083437960417917 +u16 -> 13883626507745758865 +i32 -> 14729289749324644435 +u32 -> 12103188004421096629 +i64 -> 961258395613152243 +u64 -> 15804125619400967189 +ipaddr -> 14737150884237616680 +uuid -> 16911606541498230091 +bool -> 10983724023695040909 \ No newline at end of file diff --git a/oximeter/db/test-output/sample-timeseries-key.txt b/oximeter/db/test-output/sample-timeseries-key.txt new file mode 100644 index 0000000000..aeb515c78e --- /dev/null +++ b/oximeter/db/test-output/sample-timeseries-key.txt @@ -0,0 +1 @@ +365003276793586811 \ No newline at end of file diff --git a/oximeter/oximeter/Cargo.toml b/oximeter/oximeter/Cargo.toml index 7d01b8f8be..8a69494d5a 100644 --- a/oximeter/oximeter/Cargo.toml +++ b/oximeter/oximeter/Cargo.toml @@ -13,6 +13,7 @@ omicron-common.workspace = true oximeter-macro-impl.workspace = true schemars = { workspace = true, features = [ "uuid1", "bytes", "chrono" ] } serde.workspace = true +strum.workspace = true thiserror.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/oximeter/oximeter/src/types.rs b/oximeter/oximeter/src/types.rs index aa61a426e3..d3f1b9e746 100644 --- a/oximeter/oximeter/src/types.rs +++ b/oximeter/oximeter/src/types.rs @@ -90,7 +90,15 @@ impl_field_type_from! { bool, FieldType::Bool } /// The `FieldValue` contains the value of a target or metric field. #[derive( - Clone, Debug, Hash, PartialEq, Eq, JsonSchema, Serialize, Deserialize, + Clone, + Debug, + Hash, + PartialEq, + Eq, + JsonSchema, + Serialize, + Deserialize, + strum::EnumCount, )] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum FieldValue { @@ -761,6 +769,11 @@ impl Sample { self.target.fields.values() } + /// Return the sorted fields of this sample's target. + pub fn sorted_target_fields(&self) -> &BTreeMap { + &self.target.fields + } + /// Return the name of this sample's metric. pub fn metric_name(&self) -> &str { &self.metric.name @@ -771,6 +784,11 @@ impl Sample { self.metric.fields.values() } + /// Return the sorted fields of this sample's metric + pub fn sorted_metric_fields(&self) -> &BTreeMap { + &self.metric.fields + } + // Check validity of field names for the target and metric. Currently this // just verifies there are no duplicate names between them. fn verify_field_names( From 2af5ec9fd63495a3d0aa90f7d4b1cd1b9eae17ff Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 12 Oct 2023 14:55:25 -0700 Subject: [PATCH 50/89] [oximeter] Add minimal versioning support (#4246) Provides a mechanism to explicitly version Oximeter schema used within Clickhouse, and to wipe that data if any schema-breaking changes are made. This is retroactively intended to solve the problems discussed within https://github.com/oxidecomputer/omicron/issues/4221 , where the `timeseries_key` generation was altered. --- oximeter/collector/src/lib.rs | 7 +- oximeter/db/src/client.rs | 265 +++++++++++++++++++++++- oximeter/db/src/db-replicated-init.sql | 8 + oximeter/db/src/db-single-node-init.sql | 8 + oximeter/db/src/model.rs | 8 + 5 files changed, 281 insertions(+), 15 deletions(-) diff --git a/oximeter/collector/src/lib.rs b/oximeter/collector/src/lib.rs index 6674d65ecd..b7a14cec45 100644 --- a/oximeter/collector/src/lib.rs +++ b/oximeter/collector/src/lib.rs @@ -35,6 +35,7 @@ use omicron_common::backoff; use omicron_common::FileKv; use oximeter::types::ProducerResults; use oximeter::types::ProducerResultsItem; +use oximeter_db::model::OXIMETER_VERSION; use oximeter_db::Client; use oximeter_db::DbWrite; use serde::Deserialize; @@ -455,11 +456,7 @@ impl OximeterAgent { }; let client = Client::new(db_address, &log); let replicated = client.is_oximeter_cluster().await?; - if !replicated { - client.init_single_node_db().await?; - } else { - client.init_replicated_db().await?; - } + client.initialize_db_with_version(replicated, OXIMETER_VERSION).await?; // Spawn the task for aggregating and inserting all metrics tokio::spawn(async move { diff --git a/oximeter/db/src/client.rs b/oximeter/db/src/client.rs index 8629e4b8ef..ffa5d97d52 100644 --- a/oximeter/db/src/client.rs +++ b/oximeter/db/src/client.rs @@ -25,7 +25,9 @@ use dropshot::WhichPage; use oximeter::types::Sample; use slog::debug; use slog::error; +use slog::info; use slog::trace; +use slog::warn; use slog::Logger; use std::collections::btree_map::Entry; use std::collections::BTreeMap; @@ -269,7 +271,103 @@ impl Client { .map_err(|e| Error::Database(e.to_string())) } - // Verifies if instance is part of oximeter_cluster + /// Validates that the schema used by the DB matches the version used by + /// the executable using it. + /// + /// This function will wipe metrics data if the version stored within + /// the DB is less than the schema version of Oximeter. + /// If the version in the DB is newer than what is known to Oximeter, an + /// error is returned. + /// + /// NOTE: This function is not safe for concurrent usage! + pub async fn initialize_db_with_version( + &self, + replicated: bool, + expected_version: u64, + ) -> Result<(), Error> { + info!(self.log, "reading db version"); + + // Read the version from the DB + let version = self.read_latest_version().await?; + info!(self.log, "read oximeter database version"; "version" => version); + + // Decide how to conform the on-disk version with this version of + // Oximeter. + if version < expected_version { + info!(self.log, "wiping and re-initializing oximeter schema"); + // If the on-storage version is less than the constant embedded into + // this binary, the DB is out-of-date. Drop it, and re-populate it + // later. + if !replicated { + self.wipe_single_node_db().await?; + self.init_single_node_db().await?; + } else { + self.wipe_replicated_db().await?; + self.init_replicated_db().await?; + } + } else if version > expected_version { + // If the on-storage version is greater than the constant embedded + // into this binary, we may have downgraded. + return Err(Error::Database( + format!( + "Expected version {expected_version}, saw {version}. Downgrading is not supported.", + ) + )); + } else { + // If the version matches, we don't need to update the DB + return Ok(()); + } + + info!(self.log, "inserting current version"; "version" => expected_version); + self.insert_version(expected_version).await?; + Ok(()) + } + + async fn read_latest_version(&self) -> Result { + let sql = format!( + "SELECT MAX(value) FROM {db_name}.version;", + db_name = crate::DATABASE_NAME, + ); + + let version = match self.execute_with_body(sql).await { + Ok(body) if body.is_empty() => { + warn!( + self.log, + "no version in database (treated as 'version 0')" + ); + 0 + } + Ok(body) => body.trim().parse::().map_err(|err| { + Error::Database(format!("Cannot read version: {err}")) + })?, + Err(Error::Database(err)) + // Case 1: The database has not been created. + if err.contains("Database oximeter doesn't exist") || + // Case 2: The database has been created, but it's old (exists + // prior to the version table). + err.contains("Table oximeter.version doesn't exist") => + { + warn!(self.log, "oximeter database does not exist, or is out-of-date"); + 0 + } + Err(err) => { + warn!(self.log, "failed to read version"; "error" => err.to_string()); + return Err(err); + } + }; + Ok(version) + } + + async fn insert_version(&self, version: u64) -> Result<(), Error> { + let sql = format!( + "INSERT INTO {db_name}.version (*) VALUES ({version}, now());", + db_name = crate::DATABASE_NAME, + ); + self.execute_with_body(sql).await?; + Ok(()) + } + + /// Verifies if instance is part of oximeter_cluster pub async fn is_oximeter_cluster(&self) -> Result { let sql = String::from("SHOW CLUSTERS FORMAT JSONEachRow;"); let res = self.execute_with_body(sql).await?; @@ -710,7 +808,8 @@ mod tests { // on the ubuntu CI job with "Failed to detect ClickHouse subprocess within timeout" #[ignore] async fn test_build_replicated() { - let log = slog::Logger::root(slog::Discard, o!()); + let logctx = test_setup_log("test_build_replicated"); + let log = &logctx.log; // Start all Keeper coordinator nodes let cur_dir = std::env::current_dir().unwrap(); @@ -819,11 +918,14 @@ mod tests { k3.cleanup().await.expect("Failed to cleanup ClickHouse keeper 3"); db_1.cleanup().await.expect("Failed to cleanup ClickHouse server 1"); db_2.cleanup().await.expect("Failed to cleanup ClickHouse server 2"); + + logctx.cleanup_successful(); } #[tokio::test] async fn test_client_insert() { - let log = slog::Logger::root(slog::Discard, o!()); + let logctx = test_setup_log("test_client_insert"); + let log = &logctx.log; // Let the OS assign a port and discover it after ClickHouse starts let mut db = ClickHouseInstance::new_single_node(0) @@ -845,6 +947,7 @@ mod tests { }; client.insert_samples(&samples).await.unwrap(); db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); } // This is a target with the same name as that in `lib.rs` used for other tests, but with a @@ -1307,7 +1410,8 @@ mod tests { #[tokio::test] async fn test_schema_mismatch() { - let log = slog::Logger::root(slog::Discard, o!()); + let logctx = test_setup_log("test_schema_mismatch"); + let log = &logctx.log; // Let the OS assign a port and discover it after ClickHouse starts let mut db = ClickHouseInstance::new_single_node(0) @@ -1337,11 +1441,141 @@ mod tests { let result = client.verify_sample_schema(&sample).await; assert!(matches!(result, Err(Error::SchemaMismatch { .. }))); db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); + } + + // Returns the number of timeseries schemas being used. + async fn get_schema_count(client: &Client) -> usize { + client + .execute_with_body( + "SELECT * FROM oximeter.timeseries_schema FORMAT JSONEachRow;", + ) + .await + .expect("Failed to SELECT from database") + .lines() + .count() + } + + #[tokio::test] + async fn test_database_version_update_idempotent() { + let logctx = test_setup_log("test_database_version_update_idempotent"); + let log = &logctx.log; + + let mut db = ClickHouseInstance::new_single_node(0) + .await + .expect("Failed to start ClickHouse"); + let address = SocketAddr::new("::1".parse().unwrap(), db.port()); + + let replicated = false; + + // Initialize the database... + let client = Client::new(address, &log); + client + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) + .await + .expect("Failed to initialize timeseries database"); + + // Insert data here so we can verify it still exists later. + // + // The values here don't matter much, we just want to check that + // the database data hasn't been dropped. + assert_eq!(0, get_schema_count(&client).await); + let sample = test_util::make_sample(); + client.insert_samples(&[sample.clone()]).await.unwrap(); + assert_eq!(1, get_schema_count(&client).await); + + // Re-initialize the database, see that our data still exists + client + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) + .await + .expect("Failed to initialize timeseries database"); + + assert_eq!(1, get_schema_count(&client).await); + + db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_database_version_will_not_downgrade() { + let logctx = test_setup_log("test_database_version_will_not_downgrade"); + let log = &logctx.log; + + let mut db = ClickHouseInstance::new_single_node(0) + .await + .expect("Failed to start ClickHouse"); + let address = SocketAddr::new("::1".parse().unwrap(), db.port()); + + let replicated = false; + + // Initialize the database + let client = Client::new(address, &log); + client + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) + .await + .expect("Failed to initialize timeseries database"); + + // Bump the version of the database to a "too new" version + client + .insert_version(model::OXIMETER_VERSION + 1) + .await + .expect("Failed to insert very new DB version"); + + // Expect a failure re-initializing the client. + // + // This will attempt to initialize the client with "version = + // model::OXIMETER_VERSION", which is "too old". + client + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) + .await + .expect_err("Should have failed, downgrades are not supported"); + + db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_database_version_wipes_old_version() { + let logctx = test_setup_log("test_database_version_wipes_old_version"); + let log = &logctx.log; + let mut db = ClickHouseInstance::new_single_node(0) + .await + .expect("Failed to start ClickHouse"); + let address = SocketAddr::new("::1".parse().unwrap(), db.port()); + + let replicated = false; + + // Initialize the Client + let client = Client::new(address, &log); + client + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) + .await + .expect("Failed to initialize timeseries database"); + + // Insert data here so we can remove it later. + // + // The values here don't matter much, we just want to check that + // the database data gets dropped later. + assert_eq!(0, get_schema_count(&client).await); + let sample = test_util::make_sample(); + client.insert_samples(&[sample.clone()]).await.unwrap(); + assert_eq!(1, get_schema_count(&client).await); + + // If we try to upgrade to a newer version, we'll drop old data. + client + .initialize_db_with_version(replicated, model::OXIMETER_VERSION + 1) + .await + .expect("Should have initialized database successfully"); + assert_eq!(0, get_schema_count(&client).await); + + db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); } #[tokio::test] async fn test_schema_update() { - let log = slog::Logger::root(slog::Discard, o!()); + let logctx = test_setup_log("test_schema_update"); + let log = &logctx.log; // Let the OS assign a port and discover it after ClickHouse starts let mut db = ClickHouseInstance::new_single_node(0) @@ -1415,6 +1649,7 @@ mod tests { assert_eq!(expected_schema, schema[0]); db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); } async fn setup_filter_testcase() -> (ClickHouseInstance, Client, Vec) @@ -1589,12 +1824,14 @@ mod tests { #[tokio::test] async fn test_bad_database_connection() { - let log = slog::Logger::root(slog::Discard, o!()); + let logctx = test_setup_log("test_bad_database_connection"); + let log = &logctx.log; let client = Client::new("127.0.0.1:443".parse().unwrap(), &log); assert!(matches!( client.ping().await, Err(Error::DatabaseUnavailable(_)) )); + logctx.cleanup_successful(); } #[tokio::test] @@ -1617,7 +1854,8 @@ mod tests { datum: i64, } - let log = Logger::root(slog::Discard, o!()); + let logctx = test_setup_log("test_differentiate_by_timeseries_name"); + let log = &logctx.log; // Let the OS assign a port and discover it after ClickHouse starts let db = ClickHouseInstance::new_single_node(0) @@ -1665,6 +1903,7 @@ mod tests { ); assert_eq!(timeseries.target.name, "my_target"); assert_eq!(timeseries.metric.name, "second_metric"); + logctx.cleanup_successful(); } #[derive(Debug, Clone, oximeter::Target)] @@ -1980,7 +2219,8 @@ mod tests { .await .expect("Failed to start ClickHouse"); let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let log = Logger::root(slog::Discard, o!()); + let logctx = test_setup_log("test_select_timeseries_with_start_time"); + let log = &logctx.log; let client = Client::new(address, &log); client .init_single_node_db() @@ -2015,6 +2255,7 @@ mod tests { } } db.cleanup().await.expect("Failed to cleanup database"); + logctx.cleanup_successful(); } #[tokio::test] @@ -2024,7 +2265,8 @@ mod tests { .await .expect("Failed to start ClickHouse"); let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let log = Logger::root(slog::Discard, o!()); + let logctx = test_setup_log("test_select_timeseries_with_limit"); + let log = &logctx.log; let client = Client::new(address, &log); client .init_single_node_db() @@ -2133,6 +2375,7 @@ mod tests { ); db.cleanup().await.expect("Failed to cleanup database"); + logctx.cleanup_successful(); } #[tokio::test] @@ -2142,7 +2385,8 @@ mod tests { .await .expect("Failed to start ClickHouse"); let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let log = Logger::root(slog::Discard, o!()); + let logctx = test_setup_log("test_select_timeseries_with_order"); + let log = &logctx.log; let client = Client::new(address, &log); client .init_single_node_db() @@ -2234,6 +2478,7 @@ mod tests { ); db.cleanup().await.expect("Failed to cleanup database"); + logctx.cleanup_successful(); } #[tokio::test] diff --git a/oximeter/db/src/db-replicated-init.sql b/oximeter/db/src/db-replicated-init.sql index 7b92d967af..7b756f4b0d 100644 --- a/oximeter/db/src/db-replicated-init.sql +++ b/oximeter/db/src/db-replicated-init.sql @@ -1,5 +1,13 @@ CREATE DATABASE IF NOT EXISTS oximeter ON CLUSTER oximeter_cluster; -- +CREATE TABLE IF NOT EXISTS oximeter.version ON CLUSTER oximeter_cluster +( + value UInt64, + timestamp DateTime64(9, 'UTC') +) +ENGINE = ReplicatedMergeTree() +ORDER BY (value, timestamp); +-- CREATE TABLE IF NOT EXISTS oximeter.measurements_bool_local ON CLUSTER oximeter_cluster ( timeseries_name String, diff --git a/oximeter/db/src/db-single-node-init.sql b/oximeter/db/src/db-single-node-init.sql index 5f805f5725..1f648fd5d5 100644 --- a/oximeter/db/src/db-single-node-init.sql +++ b/oximeter/db/src/db-single-node-init.sql @@ -1,5 +1,13 @@ CREATE DATABASE IF NOT EXISTS oximeter; -- +CREATE TABLE IF NOT EXISTS oximeter.version +( + value UInt64, + timestamp DateTime64(9, 'UTC') +) +ENGINE = MergeTree() +ORDER BY (value, timestamp); +-- CREATE TABLE IF NOT EXISTS oximeter.measurements_bool ( timeseries_name String, diff --git a/oximeter/db/src/model.rs b/oximeter/db/src/model.rs index 1b3b75320f..1314c5c649 100644 --- a/oximeter/db/src/model.rs +++ b/oximeter/db/src/model.rs @@ -35,6 +35,14 @@ use std::net::IpAddr; use std::net::Ipv6Addr; use uuid::Uuid; +/// Describes the version of the Oximeter database. +/// +/// See: [crate::Client::initialize_db_with_version] for usage. +/// +/// TODO(#4271): The current implementation of versioning will wipe the metrics +/// database if this number is incremented. +pub const OXIMETER_VERSION: u64 = 1; + // Wrapper type to represent a boolean in the database. // // ClickHouse's type system lacks a boolean, and using `u8` to represent them. This a safe wrapper From 078e1f0dbde62eadc12fa324146c14cfaae9574c Mon Sep 17 00:00:00 2001 From: Rain Date: Thu, 12 Oct 2023 20:11:48 -0700 Subject: [PATCH 51/89] [update-engine] a few improvements to event buffers (#4164) * For failures, track the parent key that failed (similar to aborts). * Track the last root event index that causes data for a step to be updated -- we're going to use this in the line-based displayer. * Add tests. Depends on #4163. --- update-engine/src/buffer.rs | 349 ++++++++++++++++++++++++++++------ wicket/src/ui/panes/update.rs | 22 ++- 2 files changed, 307 insertions(+), 64 deletions(-) diff --git a/update-engine/src/buffer.rs b/update-engine/src/buffer.rs index 1779ef7da6..d1028ff8cc 100644 --- a/update-engine/src/buffer.rs +++ b/update-engine/src/buffer.rs @@ -6,6 +6,7 @@ use std::{ collections::{HashMap, VecDeque}, + fmt, time::Duration, }; @@ -96,7 +97,7 @@ impl EventBuffer { /// /// This might cause older low-priority events to fall off the list. pub fn add_step_event(&mut self, event: StepEvent) { - self.event_store.handle_step_event(event, self.max_low_priority); + self.event_store.handle_root_step_event(event, self.max_low_priority); } /// Returns the root execution ID, if this event buffer is aware of any @@ -132,7 +133,8 @@ impl EventBuffer { let mut step_events = Vec::new(); let mut progress_events = Vec::new(); for (_, step_data) in self.steps().as_slice() { - step_events.extend(step_data.step_events_since(last_seen).cloned()); + step_events + .extend(step_data.step_events_since_impl(last_seen).cloned()); progress_events .extend(step_data.step_status.progress_event().cloned()); } @@ -161,7 +163,7 @@ impl EventBuffer { /// have been reported before a sender shuts down. pub fn has_pending_events_since(&self, last_seen: Option) -> bool { for (_, step_data) in self.steps().as_slice() { - if step_data.step_events_since(last_seen).next().is_some() { + if step_data.step_events_since_impl(last_seen).next().is_some() { return true; } } @@ -223,8 +225,8 @@ impl EventStore { }) } - /// Handles a step event. - fn handle_step_event( + /// Handles a non-nested step event. + fn handle_root_step_event( &mut self, event: StepEvent, max_low_priority: usize, @@ -234,12 +236,17 @@ impl EventStore { return; } + // This is a non-nested step event so the event index is a root event + // index. + let root_event_index = RootEventIndex(event.event_index); + let actions = - self.recurse_for_step_event(&event, 0, None, event.event_index); + self.recurse_for_step_event(&event, 0, None, root_event_index); if let Some(new_execution) = actions.new_execution { if new_execution.nest_level == 0 { self.root_execution_id = Some(new_execution.execution_id); } + let total_steps = new_execution.steps_to_add.len(); for (new_step_key, new_step, sort_key) in new_execution.steps_to_add { // These are brand new steps so their keys shouldn't exist in the @@ -249,6 +256,8 @@ impl EventStore { new_step, sort_key, new_execution.nest_level, + total_steps, + root_event_index, ) }); } @@ -302,7 +311,7 @@ impl EventStore { event: &StepEvent, nest_level: usize, parent_sort_key: Option<&StepSortKey>, - root_event_index: usize, + root_event_index: RootEventIndex, ) -> RecurseActions { let mut new_execution = None; let (step_key, progress_key) = match &event.kind { @@ -318,7 +327,7 @@ impl EventStore { }; let sort_key = StepSortKey::new( parent_sort_key, - root_event_index, + root_event_index.0, step.index, ); let step_node = self.add_step_node(step_key); @@ -360,7 +369,7 @@ impl EventStore { attempt_elapsed: *attempt_elapsed, }; // Mark this key and all child keys completed. - self.mark_step_key_completed(key, info); + self.mark_step_key_completed(key, info, root_event_index); // Register the next step in the event map. let next_key = StepKey { @@ -400,7 +409,7 @@ impl EventStore { attempt_elapsed: *attempt_elapsed, }; // Mark this key and all child keys completed. - self.mark_execution_id_completed(key, info); + self.mark_execution_id_completed(key, info, root_event_index); (Some(key), Some(key)) } @@ -426,7 +435,7 @@ impl EventStore { step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; - self.mark_step_failed(key, info); + self.mark_step_failed(key, info, root_event_index); (Some(key), Some(key)) } @@ -450,7 +459,7 @@ impl EventStore { step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; - self.mark_step_aborted(key, info); + self.mark_step_aborted(key, info, root_event_index); (Some(key), Some(key)) } @@ -524,11 +533,12 @@ impl EventStore { &mut self, root_key: StepKey, info: CompletionInfo, + root_event_index: RootEventIndex, ) { if let Some(value) = self.map.get_mut(&root_key) { // Completion status only applies to the root key. Nodes reachable // from this node are still marked as complete, but without status. - value.mark_completed(Some(info)); + value.mark_completed(Some(info), root_event_index); } // Mark anything reachable from this node as completed. @@ -538,7 +548,7 @@ impl EventStore { if let EventTreeNode::Step(key) = key { if key != root_key { if let Some(value) = self.map.get_mut(&key) { - value.mark_completed(None); + value.mark_completed(None, root_event_index); } } } @@ -549,10 +559,11 @@ impl EventStore { &mut self, root_key: StepKey, info: CompletionInfo, + root_event_index: RootEventIndex, ) { if let Some(value) = self.map.get_mut(&root_key) { // Completion status only applies to the root key. - value.mark_completed(Some(info)); + value.mark_completed(Some(info), root_event_index); } let mut dfs = DfsPostOrder::new( @@ -563,58 +574,87 @@ impl EventStore { if let EventTreeNode::Step(key) = key { if key != root_key { if let Some(value) = self.map.get_mut(&key) { - value.mark_completed(None); + value.mark_completed(None, root_event_index); } } } } } - fn mark_step_failed(&mut self, root_key: StepKey, info: FailureInfo) { - self.mark_step_failed_impl(root_key, |value, kind| { + fn mark_step_failed( + &mut self, + root_key: StepKey, + info: FailureInfo, + root_event_index: RootEventIndex, + ) { + self.mark_step_failed_impl(root_key, root_event_index, |value, kind| { match kind { MarkStepFailedImplKind::Root => { - value.mark_failed(Some(info.clone())); + value.mark_failed( + FailureReason::StepFailed(info.clone()), + root_event_index, + ); } MarkStepFailedImplKind::Descendant => { - value.mark_failed(None); + value.mark_failed( + FailureReason::ParentFailed { parent_step: root_key }, + root_event_index, + ); } MarkStepFailedImplKind::Future => { value.mark_will_not_be_run( WillNotBeRunReason::PreviousStepFailed { step: root_key, }, + root_event_index, ); } }; }) } - fn mark_step_aborted(&mut self, root_key: StepKey, info: AbortInfo) { - self.mark_step_failed_impl(root_key, |value, kind| { - match kind { - MarkStepFailedImplKind::Root => { - value.mark_aborted(AbortReason::StepAborted(info.clone())); - } - MarkStepFailedImplKind::Descendant => { - value.mark_aborted(AbortReason::ParentAborted { - parent_step: root_key, - }); - } - MarkStepFailedImplKind::Future => { - value.mark_will_not_be_run( - WillNotBeRunReason::PreviousStepAborted { - step: root_key, - }, - ); - } - }; - }); + fn mark_step_aborted( + &mut self, + root_key: StepKey, + info: AbortInfo, + root_event_index: RootEventIndex, + ) { + self.mark_step_failed_impl( + root_key, + root_event_index, + |value, kind| { + match kind { + MarkStepFailedImplKind::Root => { + value.mark_aborted( + AbortReason::StepAborted(info.clone()), + root_event_index, + ); + } + MarkStepFailedImplKind::Descendant => { + value.mark_aborted( + AbortReason::ParentAborted { + parent_step: root_key, + }, + root_event_index, + ); + } + MarkStepFailedImplKind::Future => { + value.mark_will_not_be_run( + WillNotBeRunReason::PreviousStepAborted { + step: root_key, + }, + root_event_index, + ); + } + }; + }, + ); } fn mark_step_failed_impl( &mut self, root_key: StepKey, + root_event_index: RootEventIndex, mut cb: impl FnMut(&mut EventBufferStepData, MarkStepFailedImplKind), ) { if let Some(value) = self.map.get_mut(&root_key) { @@ -627,7 +667,7 @@ impl EventStore { for index in 0..root_key.index { let key = StepKey { execution_id: root_key.execution_id, index }; if let Some(value) = self.map.get_mut(&key) { - value.mark_completed(None); + value.mark_completed(None, root_event_index); } } @@ -744,10 +784,17 @@ impl<'buf, S: StepSpec> EventBufferSteps<'buf, S> { pub struct EventBufferStepData { step_info: StepInfo, sort_key: StepSortKey, + // XXX: nest_level and total_steps are common to each execution, but are + // stored separately here. Should we store them in a separate map + // indexed by execution ID? nest_level: usize, - // Invariant: stored in order sorted by event_index. + total_steps: usize, + // Invariant: stored in order sorted by leaf event index. high_priority: Vec>, step_status: StepStatus, + // The last root event index that caused the data within this step to be + // updated. + last_root_event_index: RootEventIndex, } impl EventBufferStepData { @@ -755,36 +802,65 @@ impl EventBufferStepData { step_info: StepInfo, sort_key: StepSortKey, nest_level: usize, + total_steps: usize, + root_event_index: RootEventIndex, ) -> Self { Self { step_info, sort_key, nest_level, + total_steps, high_priority: Vec::new(), step_status: StepStatus::NotStarted, + last_root_event_index: root_event_index, } } + #[inline] pub fn step_info(&self) -> &StepInfo { &self.step_info } + #[inline] pub fn nest_level(&self) -> usize { self.nest_level } + #[inline] + pub fn total_steps(&self) -> usize { + self.total_steps + } + + #[inline] pub fn step_status(&self) -> &StepStatus { &self.step_status } + #[inline] + pub fn last_root_event_index(&self) -> RootEventIndex { + self.last_root_event_index + } + + #[inline] fn sort_key(&self) -> &StepSortKey { &self.sort_key } + /// Returns step events since the provided event index. + pub fn step_events_since( + &self, + last_seen: Option, + ) -> Vec<&StepEvent> { + let mut events: Vec<_> = + self.step_events_since_impl(last_seen).collect(); + events.sort_unstable_by_key(|event| event.event_index); + events + } + // Returns step events since the provided event index. // // Does not necessarily return results in sorted order. - fn step_events_since( + fn step_events_since_impl( &self, last_seen: Option, ) -> impl Iterator> { @@ -799,11 +875,12 @@ impl EventBufferStepData { iter.chain(iter2) } - fn add_high_priority_step_event(&mut self, event: StepEvent) { + fn add_high_priority_step_event(&mut self, root_event: StepEvent) { + let root_event_index = RootEventIndex(root_event.event_index); // Dedup by the *leaf index* in case nested reports aren't deduped // coming in. match self.high_priority.binary_search_by(|probe| { - probe.leaf_event_index().cmp(&event.leaf_event_index()) + probe.leaf_event_index().cmp(&root_event.leaf_event_index()) }) { Ok(_) => { // This is a duplicate. @@ -811,16 +888,19 @@ impl EventBufferStepData { Err(index) => { // index is typically the last element, so this should be quite // efficient. - self.high_priority.insert(index, event); + self.update_root_event_index(root_event_index); + self.high_priority.insert(index, root_event); } } } fn add_low_priority_step_event( &mut self, - event: StepEvent, + root_event: StepEvent, max_low_priority: usize, ) { + let root_event_index = RootEventIndex(root_event.event_index); + let mut updated = false; match &mut self.step_status { StepStatus::NotStarted => { unreachable!( @@ -831,7 +911,7 @@ impl EventBufferStepData { // Dedup by the *leaf index* in case nested reports aren't // deduped coming in. match low_priority.binary_search_by(|probe| { - probe.leaf_event_index().cmp(&event.leaf_event_index()) + probe.leaf_event_index().cmp(&root_event.leaf_event_index()) }) { Ok(_) => { // This is a duplicate. @@ -839,7 +919,8 @@ impl EventBufferStepData { Err(index) => { // The index is almost always at the end, so this is // efficient enough. - low_priority.insert(index, event); + low_priority.insert(index, root_event); + updated = true; } } @@ -857,12 +938,21 @@ impl EventBufferStepData { // likely duplicate events. } } + + if updated { + self.update_root_event_index(root_event_index); + } } - fn mark_completed(&mut self, status: Option) { + fn mark_completed( + &mut self, + status: Option, + root_event_index: RootEventIndex, + ) { match self.step_status { StepStatus::NotStarted | StepStatus::Running { .. } => { self.step_status = StepStatus::Completed { info: status }; + self.update_root_event_index(root_event_index); } StepStatus::Completed { .. } | StepStatus::Failed { .. } @@ -874,10 +964,15 @@ impl EventBufferStepData { } } - fn mark_failed(&mut self, info: Option) { + fn mark_failed( + &mut self, + reason: FailureReason, + root_event_index: RootEventIndex, + ) { match self.step_status { StepStatus::NotStarted | StepStatus::Running { .. } => { - self.step_status = StepStatus::Failed { info }; + self.step_status = StepStatus::Failed { reason }; + self.update_root_event_index(root_event_index); } StepStatus::Completed { .. } | StepStatus::Failed { .. } @@ -889,7 +984,11 @@ impl EventBufferStepData { } } - fn mark_aborted(&mut self, reason: AbortReason) { + fn mark_aborted( + &mut self, + reason: AbortReason, + root_event_index: RootEventIndex, + ) { match &mut self.step_status { StepStatus::NotStarted => { match reason { @@ -909,12 +1008,14 @@ impl EventBufferStepData { }; } } + self.update_root_event_index(root_event_index); } StepStatus::Running { progress_event, .. } => { self.step_status = StepStatus::Aborted { reason, last_progress: Some(progress_event.clone()), }; + self.update_root_event_index(root_event_index); } StepStatus::Completed { .. } | StepStatus::Failed { .. } @@ -926,10 +1027,15 @@ impl EventBufferStepData { } } - fn mark_will_not_be_run(&mut self, reason: WillNotBeRunReason) { + fn mark_will_not_be_run( + &mut self, + reason: WillNotBeRunReason, + root_event_index: RootEventIndex, + ) { match self.step_status { StepStatus::NotStarted => { self.step_status = StepStatus::WillNotBeRun { reason }; + self.update_root_event_index(root_event_index); } StepStatus::Running { .. } => { // This is a weird situation. We should never encounter it in @@ -966,6 +1072,15 @@ impl EventBufferStepData { } } } + + fn update_root_event_index(&mut self, root_event_index: RootEventIndex) { + debug_assert!( + root_event_index >= self.last_root_event_index, + "event index must be monotonically increasing" + ); + self.last_root_event_index = + self.last_root_event_index.max(root_event_index); + } } /// The step status as last seen by events. @@ -990,8 +1105,8 @@ pub enum StepStatus { /// The step has failed. Failed { - /// Failure information. - info: Option, + /// The reason for the failure. + reason: FailureReason, }, /// Execution was aborted while this step was running. @@ -1053,6 +1168,17 @@ pub struct CompletionInfo { pub attempt_elapsed: Duration, } +#[derive(Clone, Debug)] +pub enum FailureReason { + /// This step failed. + StepFailed(FailureInfo), + /// A parent step failed. + ParentFailed { + /// The parent step that failed. + parent_step: StepKey, + }, +} + #[derive(Clone, Debug)] pub struct FailureInfo { pub total_attempts: usize, @@ -1230,12 +1356,24 @@ pub struct StepKey { pub index: usize, } +/// A newtype to track root event indexes within [`EventBuffer`]s, to ensure +/// that we aren't mixing them with leaf event indexes in this code. +#[derive(Clone, Copy, Debug, Eq, PartialEq, PartialOrd, Ord, Hash)] +pub struct RootEventIndex(pub usize); + +impl fmt::Display for RootEventIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + #[cfg(test)] mod tests { use std::collections::HashSet; use anyhow::{bail, ensure, Context}; use futures::StreamExt; + use indexmap::IndexSet; use omicron_test_utils::dev::test_setup_log; use serde::{de::IntoDeserializer, Deserialize}; use tokio::sync::mpsc; @@ -1546,6 +1684,18 @@ mod tests { reported_step_events.extend(report.step_events); last_seen = report.last_seen; + // Ensure that the last root index was updated for this + // event's corresponding steps, but not for any others. + if let Event::Step(event) = event { + check_last_root_event_index(event, &buffer) + .with_context(|| { + format!( + "{description}, at index {i} (time {time}):\ + error with last root event index" + ) + })?; + } + // Call last_seen without feeding a new event in to ensure that // a report with no step events is produced. let report = buffer.generate_report_since(last_seen); @@ -1625,8 +1775,7 @@ mod tests { let mut last_seen_opt = with_deltas.then_some(None); for (i, event) in self.generated_events.iter().enumerate() { - // Going to use event_added in an upcoming commit. - let _event_added = (event_fn)(&mut buffer, event); + let event_added = (event_fn)(&mut buffer, event); let report = match last_seen_opt { Some(last_seen) => buffer.generate_report_since(last_seen), @@ -1646,6 +1795,18 @@ mod tests { }) .unwrap(); + if let Event::Step(event) = event { + if event_added { + check_last_root_event_index(event, &buffer) + .with_context(|| { + format!( + "{description}, at index {i}: \ + error with last root event index" + ) + })?; + } + } + receive_buffer.add_event_report(report.clone()); let this_step_events = receive_buffer.generate_report().step_events; @@ -1832,6 +1993,78 @@ mod tests { } } + fn check_last_root_event_index( + event: &StepEvent, + buffer: &EventBuffer, + ) -> anyhow::Result<()> { + let root_event_index = RootEventIndex(event.event_index); + let event_step_keys = step_keys(event); + let steps = buffer.steps(); + for (step_key, data) in steps.as_slice() { + let data_index = data.last_root_event_index(); + if event_step_keys.contains(step_key) { + ensure!( + data_index == root_event_index, + "last_root_event_index should have been updated \ + but wasn't (actual: {data_index}, expected: {root_event_index}) \ + for step {step_key:?} (event: {event:?})", + ); + } else { + ensure!( + data_index < root_event_index, + "last_root_event_index should *not* have been updated \ + but was (current: {data_index}, new: {root_event_index}) \ + for step {step_key:?} (event: {event:?})", + ); + } + } + + Ok(()) + } + + /// Returns the step keys that this step event would cause updates against, + /// in order from root to leaf. + fn step_keys(event: &StepEvent) -> IndexSet { + let mut out = IndexSet::new(); + step_keys_impl(event, &mut out); + out + } + + fn step_keys_impl( + event: &StepEvent, + out: &mut IndexSet, + ) { + match &event.kind { + StepEventKind::NoStepsDefined | StepEventKind::Unknown => {} + StepEventKind::ExecutionStarted { steps, .. } => { + for step in steps { + out.insert(StepKey { + execution_id: event.execution_id, + index: step.index, + }); + } + } + StepEventKind::ProgressReset { step, .. } + | StepEventKind::AttemptRetry { step, .. } + | StepEventKind::StepCompleted { step, .. } + | StepEventKind::ExecutionCompleted { last_step: step, .. } + | StepEventKind::ExecutionFailed { failed_step: step, .. } + | StepEventKind::ExecutionAborted { aborted_step: step, .. } => { + out.insert(StepKey { + execution_id: event.execution_id, + index: step.info.index, + }); + } + StepEventKind::Nested { step, event, .. } => { + out.insert(StepKey { + execution_id: event.execution_id, + index: step.info.index, + }); + step_keys_impl(event, out); + } + } + } + #[derive(Copy, Clone, Debug)] #[allow(unused)] enum WithDeltas { diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs index ea68cb4a16..da6f10cf88 100644 --- a/wicket/src/ui/panes/update.rs +++ b/wicket/src/ui/panes/update.rs @@ -29,7 +29,7 @@ use ratatui::widgets::{ use slog::{info, o, Logger}; use tui_tree_widget::{Tree, TreeItem, TreeState}; use update_engine::{ - AbortReason, ExecutionStatus, StepKey, WillNotBeRunReason, + AbortReason, ExecutionStatus, FailureReason, StepKey, WillNotBeRunReason, }; use wicket_common::update_events::{ EventBuffer, EventReport, ProgressEvent, StepOutcome, StepStatus, @@ -340,7 +340,7 @@ impl UpdatePane { Span::styled("Completed", style::successful_update_bold()), ])); } - StepStatus::Failed { info: Some(info) } => { + StepStatus::Failed { reason: FailureReason::StepFailed(info) } => { let mut spans = vec![ Span::styled("Status: ", style::selected()), Span::styled("Failed", style::failed_update_bold()), @@ -381,13 +381,23 @@ impl UpdatePane { } } } - StepStatus::Failed { info: None } => { - // No information is available, so all we can do is say that - // this step failed. - let spans = vec![ + StepStatus::Failed { + reason: FailureReason::ParentFailed { parent_step }, + } => { + let mut spans = vec![ Span::styled("Status: ", style::selected()), Span::styled("Failed", style::failed_update_bold()), ]; + if let Some(value) = id_state.event_buffer.get(parent_step) { + spans.push(Span::styled( + " at parent step ", + style::plain_text(), + )); + spans.push(Span::styled( + value.step_info().description.as_ref(), + style::selected(), + )); + } body.lines.push(Line::from(spans)); } StepStatus::Aborted { From 69113e96474c728bc7dd7d0601ed0d2134a4a2a0 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 20:12:47 -0700 Subject: [PATCH 52/89] Update Rust crate indicatif to 0.17.7 (#4249) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce17dbe311..05cad9a020 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3483,9 +3483,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.6" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b297dc40733f23a0e52728a58fa9489a5b7638a324932de16b41adc3ef80730" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" dependencies = [ "console", "instant", diff --git a/Cargo.toml b/Cargo.toml index 5c10a94706..96a6cdbacf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -210,7 +210,7 @@ hyper-rustls = "0.24.1" hyper-staticfile = "0.9.5" illumos-utils = { path = "illumos-utils" } indexmap = "2.0.0" -indicatif = { version = "0.17.6", features = ["rayon"] } +indicatif = { version = "0.17.7", features = ["rayon"] } installinator = { path = "installinator" } installinator-artifactd = { path = "installinator-artifactd" } installinator-artifact-client = { path = "clients/installinator-artifact-client" } From ee8d93b39399c8f764f67101826bc20c89003942 Mon Sep 17 00:00:00 2001 From: Greg Colombo Date: Fri, 13 Oct 2023 09:17:49 -0700 Subject: [PATCH 53/89] Use opctx_alloc in start saga to query boundary switches (#4274) Finding boundary switches in the instance start saga requires fleet query access. Use the Nexus instance allocation context to get it instead of the saga initiator's operation context. (This was introduced in #3873; it wasn't previously a problem because the instance create saga set up instance NAT state, and that saga received a boundary switch list as a parameter, which parameter was generated by using the instance allocation context. #4194 made this worse by making instance create use the start saga to start instances instead of using its own saga nodes.) Update the instance-in-silo integration test to make sure that instances created by a silo collaborator actually start. This is unfortunately not very elegant. The `instance_simulate` function family normally uses `OpContext::for_tests` to get an operation context, but that context is associated with a user that isn't in the test silo. To get around this, add some simulation interfaces that take an explicit `OpContext` and then generate one corresponding to the silo user for the test case in question. It seems like it'd be nicer to give helper routines like `instance_simulate` access to a context that is omnipotent across all silos, but I wasn't sure how best to do this. I'm definitely open to suggestions here. Tested via cargo tests. Fixes #4272. --- nexus/db-queries/src/authn/mod.rs | 4 +- nexus/src/app/sagas/instance_start.rs | 6 ++- nexus/src/app/test_interfaces.rs | 37 +++++++++++++- nexus/tests/integration_tests/instances.rs | 56 ++++++++++++++++++++-- 4 files changed, 95 insertions(+), 8 deletions(-) diff --git a/nexus/db-queries/src/authn/mod.rs b/nexus/db-queries/src/authn/mod.rs index 76824c7d08..305c359820 100644 --- a/nexus/db-queries/src/authn/mod.rs +++ b/nexus/db-queries/src/authn/mod.rs @@ -228,8 +228,8 @@ impl Context { ) } - /// Returns an authenticated context for the specific Silo user. - #[cfg(test)] + /// Returns an authenticated context for the specific Silo user. Not marked + /// as #[cfg(test)] so that this is available in integration tests. pub fn for_test_user( silo_user_id: Uuid, silo_id: Uuid, diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index 5d02d44b6b..068d2e7005 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -323,6 +323,8 @@ async fn sis_dpd_ensure( ); let datastore = osagactx.datastore(); + // Querying sleds requires fleet access; use the instance allocator context + // for this. let sled_uuid = sagactx.lookup::("sled_id")?; let (.., sled) = LookupPath::new(&osagactx.nexus().opctx_alloc, &datastore) .sled_id(sled_uuid) @@ -330,9 +332,11 @@ async fn sis_dpd_ensure( .await .map_err(ActionError::action_failed)?; + // Querying boundary switches also requires fleet access and the use of the + // instance allocator context. let boundary_switches = osagactx .nexus() - .boundary_switches(&opctx) + .boundary_switches(&osagactx.nexus().opctx_alloc) .await .map_err(ActionError::action_failed)?; diff --git a/nexus/src/app/test_interfaces.rs b/nexus/src/app/test_interfaces.rs index 486569333e..c7a6165998 100644 --- a/nexus/src/app/test_interfaces.rs +++ b/nexus/src/app/test_interfaces.rs @@ -24,6 +24,12 @@ pub trait TestInterfaces { id: &Uuid, ) -> Result>, Error>; + async fn instance_sled_by_id_with_opctx( + &self, + id: &Uuid, + opctx: &OpContext, + ) -> Result>, Error>; + /// Returns the SledAgentClient for the sled running an instance to which a /// disk is attached. async fn disk_sled_by_id( @@ -37,6 +43,12 @@ pub trait TestInterfaces { instance_id: &Uuid, ) -> Result, Error>; + async fn instance_sled_id_with_opctx( + &self, + instance_id: &Uuid, + opctx: &OpContext, + ) -> Result, Error>; + async fn set_disk_as_faulted(&self, disk_id: &Uuid) -> Result; fn set_samael_max_issue_delay(&self, max_issue_delay: chrono::Duration); @@ -52,7 +64,20 @@ impl TestInterfaces for super::Nexus { &self, id: &Uuid, ) -> Result>, Error> { - let sled_id = self.instance_sled_id(id).await?; + let opctx = OpContext::for_tests( + self.log.new(o!()), + Arc::clone(&self.db_datastore), + ); + + self.instance_sled_by_id_with_opctx(id, &opctx).await + } + + async fn instance_sled_by_id_with_opctx( + &self, + id: &Uuid, + opctx: &OpContext, + ) -> Result>, Error> { + let sled_id = self.instance_sled_id_with_opctx(id, opctx).await?; if let Some(sled_id) = sled_id { Ok(Some(self.sled_client(&sled_id).await?)) } else { @@ -83,6 +108,14 @@ impl TestInterfaces for super::Nexus { Arc::clone(&self.db_datastore), ); + self.instance_sled_id_with_opctx(id, &opctx).await + } + + async fn instance_sled_id_with_opctx( + &self, + id: &Uuid, + opctx: &OpContext, + ) -> Result, Error> { let (.., authz_instance) = LookupPath::new(&opctx, &self.db_datastore) .instance_id(*id) .lookup_for(nexus_db_queries::authz::Action::Read) @@ -90,7 +123,7 @@ impl TestInterfaces for super::Nexus { Ok(self .datastore() - .instance_fetch_with_vmm(&opctx, &authz_instance) + .instance_fetch_with_vmm(opctx, &authz_instance) .await? .sled_id()) } diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index b8fcc9f2cb..9208e21652 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -3611,10 +3611,11 @@ async fn test_instance_create_in_silo(cptestctx: &ControlPlaneTestContext) { // Create an instance using the authorization granted to the collaborator // Silo User. + let instance_name = "collaborate-with-me"; let instance_params = params::InstanceCreate { identity: IdentityMetadataCreateParams { - name: Name::try_from(String::from("ip-pool-test")).unwrap(), - description: String::from("instance to test IP Pool authz"), + name: Name::try_from(String::from(instance_name)).unwrap(), + description: String::from("instance to test creation in a silo"), }, ncpus: InstanceCpuCount::try_from(2).unwrap(), memory: ByteCount::from_gibibytes_u32(4), @@ -3635,6 +3636,34 @@ async fn test_instance_create_in_silo(cptestctx: &ControlPlaneTestContext) { .expect("Failed to create instance") .parsed_body::() .expect("Failed to parse instance"); + + // Make sure the instance can actually start even though a collaborator + // created it. + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + let authn = AuthnMode::SiloUser(user_id); + let instance_url = get_instance_url(instance_name); + let instance = instance_get_as(&client, &instance_url, authn.clone()).await; + info!(&cptestctx.logctx.log, "test got instance"; "instance" => ?instance); + assert_eq!(instance.runtime.run_state, InstanceState::Starting); + + // The default instance simulation function uses a test user that, while + // privileged, only has access to the default silo. Synthesize an operation + // context that grants access to the silo under test. + let opctx = OpContext::for_background( + cptestctx.logctx.log.new(o!()), + Arc::new(nexus_db_queries::authz::Authz::new(&cptestctx.logctx.log)), + nexus_db_queries::authn::Context::for_test_user( + user_id, + silo.identity.id, + nexus_db_queries::authn::SiloAuthnPolicy::try_from(&*DEFAULT_SILO) + .unwrap(), + ), + nexus.datastore().clone(), + ); + instance_simulate_with_opctx(nexus, &instance.identity.id, &opctx).await; + let instance = instance_get_as(&client, &instance_url, authn).await; + assert_eq!(instance.runtime.run_state, InstanceState::Running); } /// Test that appropriate OPTE V2P mappings are created and deleted. @@ -3752,9 +3781,17 @@ async fn test_instance_v2p_mappings(cptestctx: &ControlPlaneTestContext) { async fn instance_get( client: &ClientTestContext, instance_url: &str, +) -> Instance { + instance_get_as(client, instance_url, AuthnMode::PrivilegedUser).await +} + +async fn instance_get_as( + client: &ClientTestContext, + instance_url: &str, + authn_as: AuthnMode, ) -> Instance { NexusRequest::object_get(client, instance_url) - .authn_as(AuthnMode::PrivilegedUser) + .authn_as(authn_as) .execute() .await .unwrap() @@ -3857,6 +3894,19 @@ pub async fn instance_simulate(nexus: &Arc, id: &Uuid) { sa.instance_finish_transition(*id).await; } +pub async fn instance_simulate_with_opctx( + nexus: &Arc, + id: &Uuid, + opctx: &OpContext, +) { + let sa = nexus + .instance_sled_by_id_with_opctx(id, opctx) + .await + .unwrap() + .expect("instance must be on a sled to simulate a state change"); + sa.instance_finish_transition(*id).await; +} + /// Simulates state transitions for the incarnation of the instance on the /// supplied sled (which may not be the sled ID currently stored in the /// instance's CRDB record). From 02ee3c1c6c1a1d1a0ef9164bf4bfe1b65e3078d9 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 12:34:34 -0700 Subject: [PATCH 54/89] Update Rust crate atomicwrites to 0.4.2 (#4241) --- Cargo.lock | 7 +++---- Cargo.toml | 2 +- workspace-hack/Cargo.toml | 24 ++++++++---------------- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05cad9a020..10b1e82424 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -368,11 +368,11 @@ checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" [[package]] name = "atomicwrites" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1163d9d7c51de51a2b79d6df5e8888d11e9df17c752ce4a285fb6ca1580734e" +checksum = "f4d45f362125ed144544e57b0ec6de8fd6a296d41a6252fc4a20c0cf12e9ed3a" dependencies = [ - "rustix 0.37.23", + "rustix 0.38.9", "tempfile", "windows-sys 0.48.0", ] @@ -5466,7 +5466,6 @@ dependencies = [ "regex-syntax 0.7.5", "reqwest", "ring", - "rustix 0.37.23", "rustix 0.38.9", "schemars", "semver 1.0.18", diff --git a/Cargo.toml b/Cargo.toml index 96a6cdbacf..a9274d8b46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -138,7 +138,7 @@ assert_matches = "1.5.0" assert_cmd = "2.0.12" async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "1446f7e0c1f05f33a0581abd51fa873c7652ab61" } async-trait = "0.1.73" -atomicwrites = "0.4.1" +atomicwrites = "0.4.2" authz-macros = { path = "nexus/authz-macros" } backoff = { version = "0.4.0", features = [ "tokio" ] } base64 = "0.21.4" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index a91477678b..1ad6720fbd 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -215,56 +215,49 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } -rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-unknown-linux-gnu.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } -rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-apple-darwin.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } -rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-apple-darwin.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } -rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.aarch64-apple-darwin.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } -rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.aarch64-apple-darwin.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } -rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-unknown-illumos.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } -rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } toml_edit = { version = "0.19.15", features = ["serde"] } @@ -273,8 +266,7 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f hyper-rustls = { version = "0.24.1" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } -rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.9", features = ["fs", "termios"] } -rustix-d736d0ac4424f0f1 = { package = "rustix", version = "0.37.23", features = ["fs", "termios"] } +rustix = { version = "0.38.9", features = ["fs", "termios"] } toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } toml_edit = { version = "0.19.15", features = ["serde"] } From dc2c66961be81b524836782ba24c60eee410f039 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 12:59:20 -0700 Subject: [PATCH 55/89] Update Rust crate libc to 0.2.149 (#4250) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- workspace-hack/Cargo.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 10b1e82424..1f2ee1511b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3886,9 +3886,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libdlpi-sys" diff --git a/Cargo.toml b/Cargo.toml index a9274d8b46..6a671e6b49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -221,7 +221,7 @@ ipnetwork = { version = "0.20", features = ["schemars"] } itertools = "0.11.0" key-manager = { path = "key-manager" } lazy_static = "1.4.0" -libc = "0.2.148" +libc = "0.2.149" linear-map = "1.2.0" macaddr = { version = "1.0.1", features = ["serde_std"] } mime_guess = "2.0.4" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 1ad6720fbd..1871fd1b4d 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -59,7 +59,7 @@ ipnetwork = { version = "0.20.0", features = ["schemars"] } itertools = { version = "0.10.5" } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.148", features = ["extra_traits"] } +libc = { version = "0.2.149", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] } memchr = { version = "2.6.3" } @@ -157,7 +157,7 @@ ipnetwork = { version = "0.20.0", features = ["schemars"] } itertools = { version = "0.10.5" } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.148", features = ["extra_traits"] } +libc = { version = "0.2.149", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] } memchr = { version = "2.6.3" } From 474e64e11702e48682b6a5407ab21e69ef59b331 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Fri, 13 Oct 2023 22:56:38 -0500 Subject: [PATCH 56/89] Bump web console (TLS certs on silo create form, various fixes) (#4278) https://github.com/oxidecomputer/console/compare/0cc1e03a...9940fd0d * [9940fd0d](https://github.com/oxidecomputer/console/commit/9940fd0d) oxidecomputer/console#1790 * [d07ca142](https://github.com/oxidecomputer/console/commit/d07ca142) oxidecomputer/console#1784 * [0a32ccf8](https://github.com/oxidecomputer/console/commit/0a32ccf8) oxidecomputer/console#1787 * [e74d5aec](https://github.com/oxidecomputer/console/commit/e74d5aec) oxidecomputer/console#1772 * [a68c7c86](https://github.com/oxidecomputer/console/commit/a68c7c86) oxidecomputer/console#1779 * [b2b3a748](https://github.com/oxidecomputer/console/commit/b2b3a748) oxidecomputer/console#1774 * [0f015505](https://github.com/oxidecomputer/console/commit/0f015505) bump playwright for UI mode improvements * [1ccff371](https://github.com/oxidecomputer/console/commit/1ccff371) oxidecomputer/console#1582 --- tools/console_version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/console_version b/tools/console_version index 0c30c707e1..3b3ba9d652 100644 --- a/tools/console_version +++ b/tools/console_version @@ -1,2 +1,2 @@ -COMMIT="0cc1e03a24b3f5da275d15b969978a385d6b3b27" -SHA2="46a186fc3bf919a3aa2871aeab8441e4a13ed134f912b5d76c7ff891fed66cee" +COMMIT="9940fd0d14987ac56d1f4638c050c5c38c729172" +SHA2="2048b6342927996b46cb756aa1c3fb32fbf8339c9fbed6fe5896bfefe091ec08" From 43a9965970414a829314667b2096d572e056125b Mon Sep 17 00:00:00 2001 From: David Crespo Date: Sat, 14 Oct 2023 01:00:54 -0500 Subject: [PATCH 57/89] Bump web console (bump API client) (#4281) https://github.com/oxidecomputer/console/compare/9940fd0d...3538c32a * [3538c32a](https://github.com/oxidecomputer/console/commit/3538c32a) oxidecomputer/console#1791 --- tools/console_version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/console_version b/tools/console_version index 3b3ba9d652..218aef576d 100644 --- a/tools/console_version +++ b/tools/console_version @@ -1,2 +1,2 @@ -COMMIT="9940fd0d14987ac56d1f4638c050c5c38c729172" -SHA2="2048b6342927996b46cb756aa1c3fb32fbf8339c9fbed6fe5896bfefe091ec08" +COMMIT="3538c32a5189bd22df8f6a573399dacfbe81efaa" +SHA2="3289989f2cd6c71ea800e73231190455cc8e4e45ae9304293050b925a9fd9423" From 42732fa7d15878aedf226a24ecc52c0c8322c858 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Mon, 16 Oct 2023 09:12:51 -0700 Subject: [PATCH 58/89] Update Crucible and Propolis (#4282) Propolis changes: PHD: refactor & add support Propolis server "environments" (#547) Begin making Accessor interface more robust Update Crucible and Omicron deps for Hakari fixes Add cloud-init volume generation to standalone Use specified toolchain version for all GHA checks Use params to configure rust-toolchain in GHA Update and lock GHA dependencies Crucible changes: Use regions_dataset path for apply_smf (#1000) Don't unwrap when we can't create a dataset (#992) Fix tests and update log messages. (#995) Better backpressure (#990) Update Rust crate proptest to 1.3.1 (#977) Read only downstairs can skip Live Repair (#984) Update Rust crate expectorate to 1.1.0 (#975) Add trait for `ExtentInner` (#982) report backpressure in upstairs_info dtrace probe (#987) Support multiple downstairs operations in GtoS (#985) --------- Co-authored-by: Alan Hanson --- Cargo.lock | 142 +++++++++----------------------------- Cargo.toml | 14 ++-- package-manifest.toml | 12 ++-- workspace-hack/Cargo.toml | 20 ++---- 4 files changed, 50 insertions(+), 138 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f2ee1511b..3265ed19de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,17 +67,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom 0.2.10", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.3" @@ -506,7 +495,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "bhyve_api_sys", "libc", @@ -516,7 +505,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "libc", "strum", @@ -1246,7 +1235,7 @@ dependencies = [ [[package]] name = "cpuid_profile_config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "propolis", "serde", @@ -1454,7 +1443,7 @@ dependencies = [ [[package]] name = "crucible" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" dependencies = [ "aes-gcm-siv", "anyhow", @@ -1466,6 +1455,7 @@ dependencies = [ "crucible-client-types", "crucible-common", "crucible-protocol", + "crucible-workspace-hack", "dropshot", "futures", "futures-core", @@ -1493,45 +1483,45 @@ dependencies = [ "usdt", "uuid", "version_check", - "workspace-hack", ] [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" dependencies = [ "anyhow", "chrono", + "crucible-workspace-hack", "percent-encoding", "progenitor", "reqwest", "schemars", "serde", "serde_json", - "workspace-hack", ] [[package]] name = "crucible-client-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" dependencies = [ "base64 0.21.4", + "crucible-workspace-hack", "schemars", "serde", "serde_json", "uuid", - "workspace-hack", ] [[package]] name = "crucible-common" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" dependencies = [ "anyhow", "atty", + "crucible-workspace-hack", "nix 0.26.2 (registry+https://github.com/rust-lang/crates.io-index)", "rusqlite", "rustls-pemfile", @@ -1550,16 +1540,16 @@ dependencies = [ "twox-hash", "uuid", "vergen", - "workspace-hack", ] [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" dependencies = [ "anyhow", "chrono", + "crucible-workspace-hack", "percent-encoding", "progenitor", "reqwest", @@ -1567,38 +1557,43 @@ dependencies = [ "serde", "serde_json", "uuid", - "workspace-hack", ] [[package]] name = "crucible-protocol" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" dependencies = [ "anyhow", "bincode", "bytes", "crucible-common", + "crucible-workspace-hack", "num_enum 0.7.0", "schemars", "serde", "tokio-util", "uuid", - "workspace-hack", ] [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" dependencies = [ + "crucible-workspace-hack", "libc", "num-derive", "num-traits", "thiserror", - "workspace-hack", ] +[[package]] +name = "crucible-workspace-hack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd293370c6cb9c334123675263de33fc9e53bbdfc8bdd5e329237cf0205fdc7" + [[package]] name = "crunchy" version = "0.2.2" @@ -2034,7 +2029,7 @@ checksum = "7e1a8646b2c125eeb9a84ef0faa6d2d102ea0d5da60b824ade2743263117b848" [[package]] name = "dladm" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "libc", "strum", @@ -2994,9 +2989,6 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.6", -] [[package]] name = "hashbrown" @@ -3004,7 +2996,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.3", + "ahash", ] [[package]] @@ -3013,7 +3005,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ - "ahash 0.8.3", + "ahash", "allocator-api2", ] @@ -5421,7 +5413,6 @@ dependencies = [ "futures", "futures-channel", "futures-core", - "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -5429,13 +5420,11 @@ dependencies = [ "gateway-messages", "generic-array", "getrandom 0.2.10", - "hashbrown 0.12.3", "hashbrown 0.13.2", "hashbrown 0.14.0", "hex", "hyper", "hyper-rustls", - "indexmap 1.9.3", "indexmap 2.0.0", "inout", "ipnetwork", @@ -5453,9 +5442,7 @@ dependencies = [ "num-traits", "once_cell", "openapiv3", - "parking_lot 0.12.1", "petgraph", - "phf_shared 0.11.2", "postgres-types", "ppv-lite86", "predicates 3.0.4", @@ -5489,7 +5476,6 @@ dependencies = [ "toml_datetime", "toml_edit 0.19.15", "tracing", - "tracing-core", "trust-dns-proto", "unicode-bidi", "unicode-normalization", @@ -6628,7 +6614,7 @@ dependencies = [ [[package]] name = "propolis" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "anyhow", "bhyve_api", @@ -6661,7 +6647,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "async-trait", "base64 0.21.4", @@ -6685,7 +6671,7 @@ dependencies = [ [[package]] name = "propolis-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "anyhow", "async-trait", @@ -6737,7 +6723,7 @@ dependencies = [ [[package]] name = "propolis-server-config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "cpuid_profile_config", "serde", @@ -6749,7 +6735,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "schemars", "serde", @@ -9267,7 +9253,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", - "valuable", ] [[package]] @@ -9756,12 +9741,6 @@ dependencies = [ "serde", ] -[[package]] -name = "valuable" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" - [[package]] name = "vcpkg" version = "0.2.15" @@ -9796,7 +9775,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "viona_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "libc", "viona_api_sys", @@ -9805,7 +9784,7 @@ dependencies = [ [[package]] name = "viona_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=901b710b6e5bd05a94a323693c2b971e7e7b240e#901b710b6e5bd05a94a323693c2b971e7e7b240e" +source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" dependencies = [ "libc", ] @@ -10384,61 +10363,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "workspace-hack" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=20273bcca1fd5834ebc3e67dfa7020f0e99ad681#20273bcca1fd5834ebc3e67dfa7020f0e99ad681" -dependencies = [ - "bitflags 2.4.0", - "bytes", - "cc", - "chrono", - "console", - "crossbeam-utils", - "crypto-common", - "digest", - "either", - "futures-channel", - "futures-core", - "futures-executor", - "futures-sink", - "futures-util", - "getrandom 0.2.10", - "hashbrown 0.12.3", - "hex", - "hyper", - "indexmap 1.9.3", - "libc", - "log", - "mio", - "num-traits", - "once_cell", - "openapiv3", - "parking_lot 0.12.1", - "phf_shared 0.11.2", - "rand 0.8.5", - "rand_chacha 0.3.1", - "rand_core 0.6.4", - "reqwest", - "rustls", - "schemars", - "semver 1.0.18", - "serde", - "slog", - "syn 1.0.109", - "syn 2.0.32", - "time", - "time-macros", - "tokio", - "tokio-util", - "toml_datetime", - "toml_edit 0.19.15", - "tracing", - "tracing-core", - "usdt", - "uuid", -] - [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 6a671e6b49..72a7f6157e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,10 +163,10 @@ cookie = "0.16" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" } -crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } +crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } curve25519-dalek = "4" datatest-stable = "0.1.3" display-error-chain = "0.1.1" @@ -281,9 +281,9 @@ pretty-hex = "0.3.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "901b710b6e5bd05a94a323693c2b971e7e7b240e" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "901b710b6e5bd05a94a323693c2b971e7e7b240e", features = [ "generated-migration" ] } -propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "901b710b6e5bd05a94a323693c2b971e7e7b240e", default-features = false, features = ["mock-only"] } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "334df299a56cd0d33e1227ed4ce4d2fe7478d541" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "334df299a56cd0d33e1227ed4ce4d2fe7478d541", features = [ "generated-migration" ] } +propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "334df299a56cd0d33e1227ed4ce4d2fe7478d541", default-features = false, features = ["mock-only"] } proptest = "1.3.1" quote = "1.0" rand = "0.8.5" diff --git a/package-manifest.toml b/package-manifest.toml index 7cf235c24a..a88f8170d0 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -381,10 +381,10 @@ only_for_targets.image = "standard" # 3. Use source.type = "manual" instead of "prebuilt" source.type = "prebuilt" source.repo = "crucible" -source.commit = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +source.commit = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible.sha256.txt -source.sha256 = "0671570dfed8bff8e64c42a41269d961426bdd07e72b9ca8c2e3f28e7ead3c1c" +source.sha256 = "010281ff5c3a0807c9e770d79264c954816a055aa482988d81e85ed98242e454" output.type = "zone" [package.crucible-pantry] @@ -392,10 +392,10 @@ service_name = "crucible_pantry" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "20273bcca1fd5834ebc3e67dfa7020f0e99ad681" +source.commit = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-pantry.sha256.txt -source.sha256 = "c35cc24945d047f8d77e438ee606e6a83be64f0f97356fdc3308be716dcf3718" +source.sha256 = "809936edff2957e761e49667d5477e34b7a862050b4e082a59fdc95096d3bdd5" output.type = "zone" # Refer to @@ -406,10 +406,10 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "901b710b6e5bd05a94a323693c2b971e7e7b240e" +source.commit = "334df299a56cd0d33e1227ed4ce4d2fe7478d541" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "0f681cdbe7312f66fd3c99fe033b379e49c59fa4ad04d307f68b12514307e976" +source.sha256 = "531e0654de94b6e805836c35aa88b8a1ac691184000a03976e2b7825061e904e" output.type = "zone" [package.maghemite] diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 1871fd1b4d..36c3fe3f5f 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -39,7 +39,6 @@ flate2 = { version = "1.0.27" } futures = { version = "0.3.28" } futures-channel = { version = "0.3.28", features = ["sink"] } futures-core = { version = "0.3.28" } -futures-executor = { version = "0.3.28" } futures-io = { version = "0.3.28", default-features = false, features = ["std"] } futures-sink = { version = "0.3.28" } futures-task = { version = "0.3.28", default-features = false, features = ["std"] } @@ -49,11 +48,9 @@ generic-array = { version = "0.14.7", default-features = false, features = ["mor getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } -hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12.3", features = ["raw"] } hex = { version = "0.4.3", features = ["serde"] } hyper = { version = "0.14.27", features = ["full"] } -indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1.9.3", default-features = false, features = ["serde-1", "std"] } -indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2.0.0", features = ["serde"] } +indexmap = { version = "2.0.0", features = ["serde"] } inout = { version = "0.1.3", default-features = false, features = ["std"] } ipnetwork = { version = "0.20.0", features = ["schemars"] } itertools = { version = "0.10.5" } @@ -68,9 +65,7 @@ num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } openapiv3 = { version = "1.0.3", default-features = false, features = ["skip_serializing_defaults"] } -parking_lot = { version = "0.12.1", features = ["send_guard"] } petgraph = { version = "0.6.4", features = ["serde-1"] } -phf_shared = { version = "0.11.2" } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.0.4" } @@ -91,7 +86,7 @@ slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "rele spin = { version = "0.9.8" } string_cache = { version = "0.8.7" } subtle = { version = "2.5.0" } -syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } textwrap = { version = "0.16.0" } time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } @@ -100,7 +95,6 @@ tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serd tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } tracing = { version = "0.1.37", features = ["log"] } -tracing-core = { version = "0.1.31" } trust-dns-proto = { version = "0.22.0" } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } @@ -137,7 +131,6 @@ flate2 = { version = "1.0.27" } futures = { version = "0.3.28" } futures-channel = { version = "0.3.28", features = ["sink"] } futures-core = { version = "0.3.28" } -futures-executor = { version = "0.3.28" } futures-io = { version = "0.3.28", default-features = false, features = ["std"] } futures-sink = { version = "0.3.28" } futures-task = { version = "0.3.28", default-features = false, features = ["std"] } @@ -147,11 +140,9 @@ generic-array = { version = "0.14.7", default-features = false, features = ["mor getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } -hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12.3", features = ["raw"] } hex = { version = "0.4.3", features = ["serde"] } hyper = { version = "0.14.27", features = ["full"] } -indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1.9.3", default-features = false, features = ["serde-1", "std"] } -indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2.0.0", features = ["serde"] } +indexmap = { version = "2.0.0", features = ["serde"] } inout = { version = "0.1.3", default-features = false, features = ["std"] } ipnetwork = { version = "0.20.0", features = ["schemars"] } itertools = { version = "0.10.5" } @@ -166,9 +157,7 @@ num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } openapiv3 = { version = "1.0.3", default-features = false, features = ["skip_serializing_defaults"] } -parking_lot = { version = "0.12.1", features = ["send_guard"] } petgraph = { version = "0.6.4", features = ["serde-1"] } -phf_shared = { version = "0.11.2" } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.0.4" } @@ -189,7 +178,7 @@ slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "rele spin = { version = "0.9.8" } string_cache = { version = "0.8.7" } subtle = { version = "2.5.0" } -syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } textwrap = { version = "0.16.0" } time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } @@ -199,7 +188,6 @@ tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serd tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } tracing = { version = "0.1.37", features = ["log"] } -tracing-core = { version = "0.1.31" } trust-dns-proto = { version = "0.22.0" } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } From 7d8878998de09d2cd1b461c15fc2de18b7717d9b Mon Sep 17 00:00:00 2001 From: bnaecker Date: Mon, 16 Oct 2023 10:17:19 -0700 Subject: [PATCH 59/89] Create zone bundles from ZFS snapshots (#4225) - Fixes #4010 - Previously, we copied log files directly out of their original locations, which meant we contended with several other components: `logadm` rotating the log file; the log archiver moving the to longer-term storage; and the program writing to the file itself. This commit changes the operation of the bundler, to first create a ZFS snapshot of the filesystem(s) containing the log files, clone them, and then copy files out of the clones. We destroy those clones / snapshots after completing, and when the sled-agent starts to help with crash-safety. --- illumos-utils/src/running_zone.rs | 78 +--- illumos-utils/src/zfs.rs | 118 +++++ sled-agent/src/bootstrap/pre_server.rs | 2 +- sled-agent/src/zone_bundle.rs | 621 ++++++++++++++++++++----- 4 files changed, 646 insertions(+), 173 deletions(-) diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index 734f22bd30..805419cb5d 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -391,13 +391,16 @@ pub struct RunningZone { } impl RunningZone { + /// The path to the zone's root filesystem (i.e., `/`), within zonepath. + pub const ROOT_FS_PATH: &'static str = "root"; + pub fn name(&self) -> &str { &self.inner.name } - /// Returns the filesystem path to the zone's root + /// Returns the filesystem path to the zone's root in the GZ. pub fn root(&self) -> Utf8PathBuf { - self.inner.zonepath.join("root") + self.inner.zonepath.join(Self::ROOT_FS_PATH) } pub fn control_interface(&self) -> AddrObject { @@ -958,13 +961,11 @@ impl RunningZone { }; let binary = Utf8PathBuf::from(path); - // Fetch any log files for this SMF service. - let Some((log_file, rotated_log_files)) = - self.service_log_files(&service_name)? + let Some(log_file) = self.service_log_file(&service_name)? else { error!( self.inner.log, - "failed to find log files for existing service"; + "failed to find log file for existing service"; "service_name" => &service_name, ); continue; @@ -975,7 +976,6 @@ impl RunningZone { binary, pid, log_file, - rotated_log_files, }); } } @@ -992,72 +992,24 @@ impl RunningZone { .collect()) } - /// Return any SMF log files associated with the named service. + /// Return any SMF log file associated with the named service. /// - /// Given a named service, this returns a tuple of the latest or current log - /// file, and an array of any rotated log files. If the service does not - /// exist, or there are no log files, `None` is returned. - pub fn service_log_files( + /// Given a named service, this returns the path of the current log file. + /// This can be used to find rotated or archived log files, but keep in mind + /// this returns only the current, if it exists. + pub fn service_log_file( &self, name: &str, - ) -> Result)>, ServiceError> { + ) -> Result, ServiceError> { let output = self.run_cmd(&["svcs", "-L", name])?; let mut lines = output.lines(); let Some(current) = lines.next() else { return Ok(None); }; - // We need to prepend the zonepath root to get the path in the GZ. We - // can do this with `join()`, but that will _replace_ the path if the - // second one is absolute. So trim any prefixed `/` from each path. - let root = self.root(); - let current_log_file = - root.join(current.trim().trim_start_matches('/')); - - // The rotated log files should have the same prefix as the current, but - // with an index appended. We'll search the parent directory for - // matching names, skipping the current file. - // - // See https://illumos.org/man/8/logadm for details on the naming - // conventions around these files. - let dir = current_log_file.parent().unwrap(); - let mut rotated_files: Vec = Vec::new(); - for entry in dir.read_dir_utf8()? { - let entry = entry?; - let path = entry.path(); - - // Camino's Utf8Path only considers whole path components to match, - // so convert both paths into a &str and use that object's - // starts_with. See the `camino_starts_with_behaviour` test. - let path_ref: &str = path.as_ref(); - let current_log_file_ref: &str = current_log_file.as_ref(); - if path != current_log_file - && path_ref.starts_with(current_log_file_ref) - { - rotated_files.push(path.clone().into()); - } - } - - Ok(Some((current_log_file, rotated_files))) + return Ok(Some(Utf8PathBuf::from(current.trim()))); } } -#[test] -fn camino_starts_with_behaviour() { - let logfile = - Utf8PathBuf::from("/zonepath/var/svc/log/oxide-nexus:default.log"); - let rotated_logfile = - Utf8PathBuf::from("/zonepath/var/svc/log/oxide-nexus:default.log.0"); - - let logfile_as_string: &str = logfile.as_ref(); - let rotated_logfile_as_string: &str = rotated_logfile.as_ref(); - - assert!(logfile != rotated_logfile); - assert!(logfile_as_string != rotated_logfile_as_string); - - assert!(!rotated_logfile.starts_with(&logfile)); - assert!(rotated_logfile_as_string.starts_with(&logfile_as_string)); -} - impl Drop for RunningZone { fn drop(&mut self) { if let Some(_) = self.id.take() { @@ -1088,8 +1040,6 @@ pub struct ServiceProcess { pub pid: u32, /// The path for the current log file. pub log_file: Utf8PathBuf, - /// The paths for any rotated log files. - pub rotated_log_files: Vec, } /// Errors returned from [`InstalledZone::install`]. diff --git a/illumos-utils/src/zfs.rs b/illumos-utils/src/zfs.rs index 9118a9a3cd..a6af997619 100644 --- a/illumos-utils/src/zfs.rs +++ b/illumos-utils/src/zfs.rs @@ -108,6 +108,26 @@ pub struct GetValueError { err: GetValueErrorRaw, } +#[derive(Debug, thiserror::Error)] +#[error("Failed to list snapshots: {0}")] +pub struct ListSnapshotsError(#[from] crate::ExecutionError); + +#[derive(Debug, thiserror::Error)] +#[error("Failed to create snapshot '{snap_name}' from filesystem '{filesystem}': {err}")] +pub struct CreateSnapshotError { + filesystem: String, + snap_name: String, + err: crate::ExecutionError, +} + +#[derive(Debug, thiserror::Error)] +#[error("Failed to delete snapshot '{filesystem}@{snap_name}': {err}")] +pub struct DestroySnapshotError { + filesystem: String, + snap_name: String, + err: crate::ExecutionError, +} + /// Wraps commands for interacting with ZFS. pub struct Zfs {} @@ -184,6 +204,20 @@ impl Zfs { Ok(filesystems) } + /// Return the name of a dataset for a ZFS object. + /// + /// The object can either be a dataset name, or a path, in which case it + /// will be resolved to the _mounted_ ZFS dataset containing that path. + pub fn get_dataset_name(object: &str) -> Result { + let mut command = std::process::Command::new(ZFS); + let cmd = command.args(&["get", "-Hpo", "value", "name", object]); + execute(cmd) + .map(|output| { + String::from_utf8_lossy(&output.stdout).trim().to_string() + }) + .map_err(|err| ListDatasetsError { name: object.to_string(), err }) + } + /// Destroys a dataset. pub fn destroy_dataset(name: &str) -> Result<(), DestroyDatasetError> { let mut command = std::process::Command::new(PFEXEC); @@ -379,6 +413,7 @@ impl Zfs { } } + /// Set the value of an Oxide-managed ZFS property. pub fn set_oxide_value( filesystem_name: &str, name: &str, @@ -404,6 +439,7 @@ impl Zfs { Ok(()) } + /// Get the value of an Oxide-managed ZFS property. pub fn get_oxide_value( filesystem_name: &str, name: &str, @@ -434,6 +470,88 @@ impl Zfs { } Ok(value.to_string()) } + + /// List all extant snapshots. + pub fn list_snapshots() -> Result, ListSnapshotsError> { + let mut command = std::process::Command::new(ZFS); + let cmd = command.args(&["list", "-H", "-o", "name", "-t", "snapshot"]); + execute(cmd) + .map(|output| { + let stdout = String::from_utf8_lossy(&output.stdout); + stdout + .trim() + .lines() + .map(|line| { + let (filesystem, snap_name) = + line.split_once('@').unwrap(); + Snapshot { + filesystem: filesystem.to_string(), + snap_name: snap_name.to_string(), + } + }) + .collect() + }) + .map_err(ListSnapshotsError::from) + } + + /// Create a snapshot of a filesystem. + /// + /// A list of properties, as name-value tuples, may be passed to this + /// method, for creating properties directly on the snapshots. + pub fn create_snapshot<'a>( + filesystem: &'a str, + snap_name: &'a str, + properties: &'a [(&'a str, &'a str)], + ) -> Result<(), CreateSnapshotError> { + let mut command = std::process::Command::new(ZFS); + let mut cmd = command.arg("snapshot"); + for (name, value) in properties.iter() { + cmd = cmd.arg("-o").arg(&format!("{name}={value}")); + } + cmd.arg(&format!("{filesystem}@{snap_name}")); + execute(cmd).map(|_| ()).map_err(|err| CreateSnapshotError { + filesystem: filesystem.to_string(), + snap_name: snap_name.to_string(), + err, + }) + } + + /// Destroy a named snapshot of a filesystem. + pub fn destroy_snapshot( + filesystem: &str, + snap_name: &str, + ) -> Result<(), DestroySnapshotError> { + let mut command = std::process::Command::new(ZFS); + let path = format!("{filesystem}@{snap_name}"); + let cmd = command.args(&["destroy", &path]); + execute(cmd).map(|_| ()).map_err(|err| DestroySnapshotError { + filesystem: filesystem.to_string(), + snap_name: snap_name.to_string(), + err, + }) + } +} + +/// A read-only snapshot of a ZFS filesystem. +#[derive(Clone, Debug)] +pub struct Snapshot { + pub filesystem: String, + pub snap_name: String, +} + +impl Snapshot { + /// Return the full path to the snapshot directory within the filesystem. + pub fn full_path(&self) -> Result { + let mountpoint = Zfs::get_value(&self.filesystem, "mountpoint")?; + Ok(Utf8PathBuf::from(mountpoint) + .join(format!(".zfs/snapshot/{}", self.snap_name))) + } +} + +impl fmt::Display for Snapshot { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}@{}", self.filesystem, self.snap_name) + } } /// Returns all datasets managed by Omicron diff --git a/sled-agent/src/bootstrap/pre_server.rs b/sled-agent/src/bootstrap/pre_server.rs index 71325fef3d..0c19c30865 100644 --- a/sled-agent/src/bootstrap/pre_server.rs +++ b/sled-agent/src/bootstrap/pre_server.rs @@ -368,7 +368,7 @@ fn ensure_zfs_key_directory_exists(log: &Logger) -> Result<(), StartError> { } fn ensure_zfs_ramdisk_dataset() -> Result<(), StartError> { - let zoned = true; + let zoned = false; let do_format = true; let encryption_details = None; let quota = None; diff --git a/sled-agent/src/zone_bundle.rs b/sled-agent/src/zone_bundle.rs index 4c2d6a4113..55661371c3 100644 --- a/sled-agent/src/zone_bundle.rs +++ b/sled-agent/src/zone_bundle.rs @@ -17,6 +17,17 @@ use chrono::Utc; use flate2::bufread::GzDecoder; use illumos_utils::running_zone::is_oxide_smf_log_file; use illumos_utils::running_zone::RunningZone; +use illumos_utils::running_zone::ServiceProcess; +use illumos_utils::zfs::CreateSnapshotError; +use illumos_utils::zfs::DestroyDatasetError; +use illumos_utils::zfs::DestroySnapshotError; +use illumos_utils::zfs::EnsureFilesystemError; +use illumos_utils::zfs::GetValueError; +use illumos_utils::zfs::ListDatasetsError; +use illumos_utils::zfs::ListSnapshotsError; +use illumos_utils::zfs::SetValueError; +use illumos_utils::zfs::Snapshot; +use illumos_utils::zfs::Zfs; use illumos_utils::zfs::ZFS; use illumos_utils::zone::AdmError; use schemars::JsonSchema; @@ -141,6 +152,68 @@ impl ZoneBundleMetadata { } } +// The name of the snapshot created from the zone root filesystem. +const ZONE_ROOT_SNAPSHOT_NAME: &'static str = "zone-root"; + +// The prefix for all the snapshots for each filesystem containing archived +// logs. Each debug data, such as `oxp_/crypt/debug`, generates a snapshot +// named `zone-archives-`. +const ARCHIVE_SNAPSHOT_PREFIX: &'static str = "zone-archives-"; + +// An extra ZFS user property attached to all zone bundle snapshots. +// +// This is used to ensure that we are not accidentally deleting ZFS objects that +// a user has created, but which happen to be named the same thing. +const ZONE_BUNDLE_ZFS_PROPERTY_NAME: &'static str = "oxide:for-zone-bundle"; +const ZONE_BUNDLE_ZFS_PROPERTY_VALUE: &'static str = "true"; + +// Initialize the ZFS resources we need for zone bundling. +// +// This deletes any snapshots matching the names we expect to create ourselves +// during bundling. +#[cfg(not(test))] +fn initialize_zfs_resources(log: &Logger) -> Result<(), BundleError> { + let zb_snapshots = + Zfs::list_snapshots().unwrap().into_iter().filter(|snap| { + // Check for snapshots named how we expect to create them. + if snap.snap_name != ZONE_ROOT_SNAPSHOT_NAME + || !snap.snap_name.starts_with(ARCHIVE_SNAPSHOT_PREFIX) + { + return false; + } + + // Additionally check for the zone-bundle-specific property. + // + // If we find a dataset that matches our names, but which _does not_ + // have such a property, we'll panic rather than possibly deleting + // user data. + let name = snap.to_string(); + let value = + Zfs::get_oxide_value(&name, ZONE_BUNDLE_ZFS_PROPERTY_NAME) + .unwrap_or_else(|_| { + panic!( + "Found a ZFS snapshot with a name reserved for \ + zone bundling, but which does not have the \ + zone-bundle-specific property. Bailing out, \ + rather than risking deletion of user data. \ + snap_name = {}, property = {}", + &name, ZONE_BUNDLE_ZFS_PROPERTY_NAME + ) + }); + assert_eq!(value, ZONE_BUNDLE_ZFS_PROPERTY_VALUE); + true + }); + for snapshot in zb_snapshots { + Zfs::destroy_snapshot(&snapshot.filesystem, &snapshot.snap_name)?; + debug!( + log, + "destroyed pre-existing zone bundle snapshot"; + "snapshot" => %snapshot, + ); + } + Ok(()) +} + /// A type managing zone bundle creation and automatic cleanup. #[derive(Clone)] pub struct ZoneBundler { @@ -256,6 +329,11 @@ impl ZoneBundler { resources: StorageResources, cleanup_context: CleanupContext, ) -> Self { + // This is compiled out in tests because there's no way to set our + // expectations on the mockall object it uses internally. Not great. + #[cfg(not(test))] + initialize_zfs_resources(&log) + .expect("Failed to initialize existing ZFS resources"); let notify_cleanup = Arc::new(Notify::new()); let inner = Arc::new(Mutex::new(Inner { resources, @@ -358,7 +436,6 @@ impl ZoneBundler { .all_u2_mountpoints(sled_hardware::disk::U2_DEBUG_DATASET) .await .into_iter() - .map(|p| p.join(zone.name())) .collect(); let context = ZoneBundleContext { cause, storage_dirs, extra_log_dirs }; info!( @@ -446,7 +523,7 @@ struct ZoneBundleContext { storage_dirs: Vec, // The reason or cause for creating a zone bundle. cause: ZoneBundleCause, - // Extra directories searched for logfiles for the name zone. + // Extra directories searched for logfiles for the named zone. // // Logs are periodically archived out of their original location, and onto // one or more U.2 drives. This field is used to specify that archive @@ -572,6 +649,30 @@ pub enum BundleError { #[error("Cleanup failed")] Cleanup(#[source] anyhow::Error), + + #[error("Failed to create ZFS snapshot")] + CreateSnapshot(#[from] CreateSnapshotError), + + #[error("Failed to destroy ZFS snapshot")] + DestroySnapshot(#[from] DestroySnapshotError), + + #[error("Failed to list ZFS snapshots")] + ListSnapshot(#[from] ListSnapshotsError), + + #[error("Failed to ensure ZFS filesystem")] + EnsureFilesystem(#[from] EnsureFilesystemError), + + #[error("Failed to destroy ZFS dataset")] + DestroyDataset(#[from] DestroyDatasetError), + + #[error("Failed to list ZFS datasets")] + ListDatasets(#[from] ListDatasetsError), + + #[error("Failed to set Oxide-specific ZFS property")] + SetProperty(#[from] SetValueError), + + #[error("Failed to get ZFS property value")] + GetProperty(#[from] GetValueError), } // Helper function to write an array of bytes into the tar archive, with @@ -597,6 +698,256 @@ fn insert_data( }) } +// Create a read-only snapshot from an existing filesystem. +fn create_snapshot( + log: &Logger, + filesystem: &str, + snap_name: &str, +) -> Result { + Zfs::create_snapshot( + filesystem, + snap_name, + &[(ZONE_BUNDLE_ZFS_PROPERTY_NAME, ZONE_BUNDLE_ZFS_PROPERTY_VALUE)], + )?; + debug!( + log, + "created snapshot"; + "filesystem" => filesystem, + "snap_name" => snap_name, + ); + Ok(Snapshot { + filesystem: filesystem.to_string(), + snap_name: snap_name.to_string(), + }) +} + +// Create snapshots for the filesystems we need to copy out all log files. +// +// A key feature of the zone-bundle process is that we pull all the log files +// for a zone. This is tricky. The logs are both being written to by the +// programs we're interested in, and also potentially being rotated by `logadm`, +// and / or archived out to the U.2s through the code in `crate::dump_setup`. +// +// We need to capture all these logs, while avoiding inconsistent state (e.g., a +// missing log message that existed when the bundle was created) and also +// interrupting the rotation and archival processes. We do this by taking ZFS +// snapshots of the relevant datasets when we want to create the bundle. +// +// When we receive a bundling request, we take a snapshot of a few datasets: +// +// - The zone filesystem itself, `oxz_/crypt/zone/`. +// +// - All of the U.2 debug datasets, like `oxp_/crypt/debug`, which we know +// contain logs for the given zone. This is done by looking at all the service +// processes in the zone, and mapping the locations of archived logs, such as +// `/pool/ext//crypt/debug/` to the zpool name. +// +// This provides us with a consistent view of the log files at the time the +// bundle was requested. Note that this ordering, taking the root FS snapshot +// first followed by the archive datasets, ensures that we don't _miss_ log +// messages that existed when the bundle was requested. It's possible that we +// double-count them however: the archiver could run concurrently, and result in +// a log file existing on the root snapshot when we create it, and also on the +// achive snapshot by the time we get around to creating that. +// +// At this point, we operate entirely on those snapshots. We search for +// "current" log files in the root snapshot, and archived log files in the +// archive snapshots. +fn create_zfs_snapshots( + log: &Logger, + zone: &RunningZone, + extra_log_dirs: &[Utf8PathBuf], +) -> Result, BundleError> { + // Snapshot the root filesystem. + let dataset = Zfs::get_dataset_name(zone.root().as_str())?; + let root_snapshot = + create_snapshot(log, &dataset, ZONE_ROOT_SNAPSHOT_NAME)?; + let mut snapshots = vec![root_snapshot]; + + // Look at all the provided extra log directories, and take a snapshot for + // any that have a directory with the zone name. These may not have any log + // file in them yet, but we'll snapshot now and then filter more judiciously + // when we actually find the files we want to bundle. + let mut maybe_err = None; + for dir in extra_log_dirs.iter() { + let zone_dir = dir.join(zone.name()); + match std::fs::metadata(&zone_dir) { + Ok(d) => { + if d.is_dir() { + let dataset = match Zfs::get_dataset_name(zone_dir.as_str()) + { + Ok(ds) => Utf8PathBuf::from(ds), + Err(e) => { + error!( + log, + "failed to list datasets, will \ + unwind any previously created snapshots"; + "error" => ?e, + ); + assert!(maybe_err + .replace(BundleError::from(e)) + .is_none()); + break; + } + }; + + // These datasets are named like `/...`. Since + // we're snapshotting zero or more of them, we disambiguate + // with the pool name. + let pool_name = dataset + .components() + .next() + .expect("Zone archive datasets must be non-empty"); + let snap_name = + format!("{}{}", ARCHIVE_SNAPSHOT_PREFIX, pool_name); + match create_snapshot(log, dataset.as_str(), &snap_name) { + Ok(snapshot) => snapshots.push(snapshot), + Err(e) => { + error!( + log, + "failed to create snapshot, will \ + unwind any previously created"; + "error" => ?e, + ); + assert!(maybe_err.replace(e).is_none()); + break; + } + } + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + trace!( + log, + "skipping non-existent zone-bundle directory"; + "dir" => %zone_dir, + ); + } + Err(e) => { + error!( + log, + "failed to get metadata for potential zone directory"; + "zone_dir" => %zone_dir, + "error" => ?e, + ); + } + } + } + if let Some(err) = maybe_err { + cleanup_zfs_snapshots(log, &snapshots); + return Err(err); + }; + Ok(snapshots) +} + +// Destroy any created ZFS snapshots. +fn cleanup_zfs_snapshots(log: &Logger, snapshots: &[Snapshot]) { + for snapshot in snapshots.iter() { + match Zfs::destroy_snapshot(&snapshot.filesystem, &snapshot.snap_name) { + Ok(_) => debug!( + log, + "destroyed zone bundle ZFS snapshot"; + "snapshot" => %snapshot, + ), + Err(e) => error!( + log, + "failed to destroy zone bundle ZFS snapshot"; + "snapshot" => %snapshot, + "error" => ?e, + ), + } + } +} + +// List all log files (current, rotated, and archived) that should be part of +// the zone bundle for a single service. +async fn find_service_log_files( + log: &Logger, + zone_name: &str, + svc: &ServiceProcess, + extra_log_dirs: &[Utf8PathBuf], + snapshots: &[Snapshot], +) -> Result, BundleError> { + // The current and any rotated, but not archived, log files live in the zone + // root filesystem. Extract any which match. + // + // There are a few path tricks to keep in mind here. We've created a + // snapshot from the zone's filesystem, which is usually something like + // `/crypt/zone/`. That snapshot is placed in a hidden + // directory within the base dataset, something like + // `oxp_/crypt/zone/.zfs/snapshot/`. + // + // The log files themselves are things like `/var/svc/log/...`, but in the + // actual ZFS dataset comprising the root FS for the zone, there are + // additional directories, most notably `/root`. So the _cloned_ + // log file will live at + // `//crypt/zone/.zfs/snapshot//root/var/svc/log/...`. + let mut current_log_file = snapshots[0].full_path()?; + current_log_file.push(RunningZone::ROOT_FS_PATH); + current_log_file.push(svc.log_file.as_str().trim_start_matches('/')); + let log_dir = + current_log_file.parent().expect("Current log file must have a parent"); + let mut log_files = vec![current_log_file.clone()]; + for entry in log_dir.read_dir_utf8().map_err(|err| { + BundleError::ReadDirectory { directory: log_dir.into(), err } + })? { + let entry = entry.map_err(|err| BundleError::ReadDirectory { + directory: log_dir.into(), + err, + })?; + let path = entry.path(); + + // Camino's Utf8Path only considers whole path components to match, + // so convert both paths into a &str and use that object's + // starts_with. See the `camino_starts_with_behaviour` test. + let path_ref: &str = path.as_ref(); + let current_log_file_ref: &str = current_log_file.as_ref(); + if path != current_log_file + && path_ref.starts_with(current_log_file_ref) + { + log_files.push(path.clone().into()); + } + } + + // The _archived_ log files are slightly trickier. They can technically live + // in many different datasets, because the archive process may need to start + // archiving to one location, but move to another if a quota is hit. We'll + // iterate over all the extra log directories and try to find any log files + // in those filesystem snapshots. + let snapped_extra_log_dirs = snapshots + .iter() + .skip(1) + .flat_map(|snapshot| { + extra_log_dirs.iter().map(|d| { + // Join the snapshot path with both the log directory and the + // zone name, to arrive at something like: + // /path/to/dataset/.zfs/snapshot//path/to/extra/ + snapshot.full_path().map(|p| p.join(d).join(zone_name)) + }) + }) + .collect::, _>>()?; + debug!( + log, + "looking for extra log files in filesystem snapshots"; + "extra_dirs" => ?&snapped_extra_log_dirs, + ); + log_files.extend( + find_archived_log_files( + log, + zone_name, + &svc.service_name, + svc.log_file.file_name().unwrap(), + snapped_extra_log_dirs.iter(), + ) + .await, + ); + debug!( + log, + "found log files"; + "log_files" => ?&log_files, + ); + Ok(log_files) +} + // Create a service bundle for the provided zone. // // This runs a series of debugging commands in the zone, to collect data about @@ -702,14 +1053,8 @@ async fn create( } } - // Debugging commands run on the specific processes this zone defines. - const ZONE_PROCESS_COMMANDS: [&str; 3] = [ - "pfiles", "pstack", - "pargs", - // TODO-completeness: We may want `gcore`, since that encompasses - // the above commands and much more. It seems like overkill now, - // however. - ]; + // Enumerate the list of Oxide-specific services inside the zone that we + // want to include in the bundling process. let procs = match zone .service_processes() .context("failed to enumerate zone service processes") @@ -733,6 +1078,48 @@ async fn create( return Err(BundleError::from(e)); } }; + + // Create ZFS snapshots of filesystems containing log files. + // + // We need to capture log files from two kinds of locations: + // + // - The zone root filesystem, where the current and rotated (but not + // archived) log files live. + // - Zero or more filesystems on the U.2s used for archiving older log + // files. + // + // Both of these are dynamic. The current log file is likely being written + // by the service itself, and `logadm` may also be rotating files. At the + // same time, the log-archival process in `dump_setup.rs` may be copying + // these out to the U.2s, after which it deletes those on the zone + // filesystem itself. + // + // To avoid various kinds of corruption, such as a bad tarball or missing + // log messages, we'll create ZFS snapshots of each of these relevant + // filesystems, and insert those (now-static) files into the zone-bundle + // tarballs. + let snapshots = + match create_zfs_snapshots(log, zone, &context.extra_log_dirs) { + Ok(snapshots) => snapshots, + Err(e) => { + error!( + log, + "failed to create ZFS snapshots"; + "zone_name" => zone.name(), + "error" => ?e, + ); + return Err(e); + } + }; + + // Debugging commands run on the specific processes this zone defines. + const ZONE_PROCESS_COMMANDS: [&str; 3] = [ + "pfiles", "pstack", + "pargs", + // TODO-completeness: We may want `gcore`, since that encompasses + // the above commands and much more. It seems like overkill now, + // however. + ]; for svc in procs.into_iter() { let pid_s = svc.pid.to_string(); for cmd in ZONE_PROCESS_COMMANDS { @@ -765,72 +1152,61 @@ async fn create( } } - // We may need to extract log files that have been archived out of the - // zone filesystem itself. See `crate::dump_setup` for the logic which - // does this. - let archived_log_files = find_archived_log_files( + // Collect and insert all log files. + // + // This takes files from the snapshot of either the zone root + // filesystem, or the filesystem containing the archived log files. + let all_log_files = match find_service_log_files( log, zone.name(), - &svc.service_name, + &svc, &context.extra_log_dirs, + &snapshots, ) - .await; - - // Copy any log files, current and rotated, into the tarball as - // well. - // - // Safety: This pathbuf was retrieved by locating an existing file - // on the filesystem, so we're sure it has a name and the unwrap is - // safe. - debug!( - log, - "appending current log file to zone bundle"; - "zone" => zone.name(), - "log_file" => %svc.log_file, - ); - if let Err(e) = builder.append_path_with_name( - &svc.log_file, - svc.log_file.file_name().unwrap(), - ) { - error!( - log, - "failed to append current log file to zone bundle"; - "zone" => zone.name(), - "log_file" => %svc.log_file, - "error" => ?e, - ); - return Err(BundleError::AddBundleData { - tarball_path: svc.log_file.file_name().unwrap().into(), - err: e, - }); - } - for f in svc.rotated_log_files.iter().chain(archived_log_files.iter()) { - debug!( - log, - "appending rotated log file to zone bundle"; - "zone" => zone.name(), - "log_file" => %f, - ); - if let Err(e) = - builder.append_path_with_name(f, f.file_name().unwrap()) - { + .await + { + Ok(f) => f, + Err(e) => { error!( log, - "failed to append rotated log file to zone bundle"; + "failed to find service log files"; "zone" => zone.name(), - "log_file" => %f, "error" => ?e, ); - return Err(BundleError::AddBundleData { - tarball_path: f.file_name().unwrap().into(), - err: e, - }); + cleanup_zfs_snapshots(&log, &snapshots); + return Err(e); + } + }; + for log_file in all_log_files.into_iter() { + match builder + .append_path_with_name(&log_file, log_file.file_name().unwrap()) + { + Ok(_) => { + debug!( + log, + "appended log file to zone bundle"; + "zone" => zone.name(), + "log_file" => %log_file, + ); + } + Err(e) => { + error!( + log, + "failed to append log file to zone bundle"; + "zone" => zone.name(), + "log_file" => %svc.log_file, + "error" => ?e, + ); + } } } } // Finish writing out the tarball itself. - builder.into_inner().context("Failed to build bundle")?; + if let Err(e) = builder.into_inner().context("Failed to build bundle") { + cleanup_zfs_snapshots(&log, &snapshots); + return Err(BundleError::from(e)); + } // Copy the bundle to the other locations. We really want the bundles to // be duplicates, not an additional, new bundle. @@ -840,14 +1216,21 @@ async fn create( // the final locations should that last copy fail for any of them. // // See: https://github.com/oxidecomputer/omicron/issues/3876. + let mut copy_err = None; for other_dir in zone_bundle_dirs.iter().skip(1) { let to = other_dir.join(&filename); debug!(log, "copying bundle"; "from" => %full_path, "to" => %to); - tokio::fs::copy(&full_path, &to).await.map_err(|err| { + if let Err(e) = tokio::fs::copy(&full_path, &to).await.map_err(|err| { BundleError::CopyArchive { from: full_path.to_owned(), to, err } - })?; + }) { + copy_err = Some(e); + break; + } + } + cleanup_zfs_snapshots(&log, &snapshots); + if let Some(err) = copy_err { + return Err(err); } - info!(log, "finished zone bundle"; "metadata" => ?zone_metadata); Ok(zone_metadata) } @@ -857,11 +1240,12 @@ async fn create( // // Note that errors are logged, rather than failing the whole function, so that // one failed listing does not prevent collecting any other log files. -async fn find_archived_log_files( +async fn find_archived_log_files<'a, T: Iterator>( log: &Logger, zone_name: &str, svc_name: &str, - dirs: &[Utf8PathBuf], + log_file_prefix: &str, + dirs: T, ) -> Vec { // The `dirs` should be things like // `/pool/ext//crypt/debug/`, but it's really up to @@ -870,7 +1254,7 @@ async fn find_archived_log_files( // Within that, we'll just look for things that appear to be Oxide-managed // SMF service log files. let mut files = Vec::new(); - for dir in dirs.iter() { + for dir in dirs { if dir.exists() { let mut rd = match tokio::fs::read_dir(&dir).await { Ok(rd) => rd, @@ -900,8 +1284,9 @@ async fn find_archived_log_files( }; let fname = path.file_name().unwrap(); let is_oxide = is_oxide_smf_log_file(fname); - let contains = fname.contains(svc_name); - if is_oxide && contains { + let matches_log_file = + fname.starts_with(log_file_prefix); + if is_oxide && matches_log_file { debug!( log, "found archived log file"; @@ -911,12 +1296,14 @@ async fn find_archived_log_files( ); files.push(path); } else { - debug!( + trace!( log, "skipping non-matching log file"; + "zone_name" => zone_name, + "service_name" => svc_name, "filename" => fname, "is_oxide_smf_log_file" => is_oxide, - "contains_svc_name" => contains, + "matches_log_file" => matches_log_file, ); } } @@ -2299,43 +2686,61 @@ mod illumos_tests { let log = test_logger(); let tmpdir = tempfile::tempdir().expect("Failed to make tempdir"); - let mut should_match = [ - "oxide-foo:default.log", - "oxide-foo:default.log.1000", - "system-illumos-foo:default.log", - "system-illumos-foo:default.log.100", - ]; - let should_not_match = [ - "oxide-foo:default", - "not-oxide-foo:default.log.1000", - "system-illumos-foo", - "not-system-illumos-foo:default.log.100", - ]; - for name in should_match.iter().chain(should_not_match.iter()) { - let path = tmpdir.path().join(name); - tokio::fs::File::create(path) - .await - .expect("failed to create dummy file"); + for prefix in ["oxide", "system-illumos"] { + let mut should_match = [ + format!("{prefix}-foo:default.log"), + format!("{prefix}-foo:default.log.1000"), + ]; + let should_not_match = [ + format!("{prefix}-foo:default"), + format!("not-{prefix}-foo:default.log.1000"), + ]; + for name in should_match.iter().chain(should_not_match.iter()) { + let path = tmpdir.path().join(name); + tokio::fs::File::create(path) + .await + .expect("failed to create dummy file"); + } + + let path = Utf8PathBuf::try_from( + tmpdir.path().as_os_str().to_str().unwrap(), + ) + .unwrap(); + let mut files = find_archived_log_files( + &log, + "zone-name", // unused here, for logging only + "svc:/oxide/foo:default", + &format!("{prefix}-foo:default.log"), + std::iter::once(&path), + ) + .await; + + // Sort everything to compare correctly. + should_match.sort(); + files.sort(); + assert_eq!(files.len(), should_match.len()); + assert!(files + .iter() + .zip(should_match.iter()) + .all(|(file, name)| { file.file_name().unwrap() == *name })); } + } - let path = - Utf8PathBuf::try_from(tmpdir.path().as_os_str().to_str().unwrap()) - .unwrap(); - let mut files = find_archived_log_files( - &log, - "zone-name", // unused here, for logging only - "foo", - &[path], - ) - .await; - - // Sort everything to compare correctly. - should_match.sort(); - files.sort(); - assert_eq!(files.len(), should_match.len()); - assert!(files - .iter() - .zip(should_match.iter()) - .all(|(file, name)| { file.file_name().unwrap() == *name })); + #[test] + fn camino_starts_with_behaviour() { + let logfile = + Utf8PathBuf::from("/zonepath/var/svc/log/oxide-nexus:default.log"); + let rotated_logfile = Utf8PathBuf::from( + "/zonepath/var/svc/log/oxide-nexus:default.log.0", + ); + + let logfile_as_string: &str = logfile.as_ref(); + let rotated_logfile_as_string: &str = rotated_logfile.as_ref(); + + assert!(logfile != rotated_logfile); + assert!(logfile_as_string != rotated_logfile_as_string); + + assert!(!rotated_logfile.starts_with(&logfile)); + assert!(rotated_logfile_as_string.starts_with(&logfile_as_string)); } } From e74f5c960dbfd77b68733f11ee35887ee5876cef Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Wed, 18 Oct 2023 07:47:58 -0700 Subject: [PATCH 60/89] Add RoT v1.0.2 prod/rel (#4284) --- .github/buildomat/jobs/tuf-repo.sh | 2 +- tools/dvt_dock_version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index e169bebff6..57d3ba8a1f 100644 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -220,7 +220,7 @@ EOF } # usage: SERIES ROT_DIR ROT_VERSION BOARDS... add_hubris_artifacts rot-staging-dev staging/dev cert-staging-dev-v1.0.2 "${ALL_BOARDS[@]}" -add_hubris_artifacts rot-prod-rel prod/rel cert-prod-rel-v1.0.0 "${ALL_BOARDS[@]}" +add_hubris_artifacts rot-prod-rel prod/rel cert-prod-rel-v1.0.2 "${ALL_BOARDS[@]}" for series in "${SERIES_LIST[@]}"; do /work/tufaceous assemble --no-generate-key /work/manifest-"$series".toml /work/repo-"$series".zip diff --git a/tools/dvt_dock_version b/tools/dvt_dock_version index 790bd3ec26..b52dded7d0 100644 --- a/tools/dvt_dock_version +++ b/tools/dvt_dock_version @@ -1 +1 @@ -COMMIT=7cbfa19bad077a3c42976357a317d18291533ba2 +COMMIT=9cb2b40cea90ad40f5c0d2c3da96d26913253e06 From 5346472a1714ff9097c925abfdaa27b1f1204070 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 18 Oct 2023 09:00:35 -0700 Subject: [PATCH 61/89] Proxy Nexus's external API out the tech port via wicketd (#4224) This implements one of the proposed solutions in RFD 431 (which I still need to get back to and refine a bit!): * Nexus gains another dropshot instance that serves the external API (using the same shared server context as the existing external API servers) on the underlay network * wicketd runs a TCP proxy on the tech port * When wicketd receives a new connection on that proxy socket, it: * looks up Nexus in internal DNS (these records already exist for the _internal_ API, but that's good enough for us: we're just after the IP address here) * connects to any Nexus instance reported by the DNS query * proxies the connection between the tech port client and the Nexus instance This is therefore exposing the "true" Nexus API, which still requires TLS, client auth, etc. --- Cargo.lock | 4 +- common/src/address.rs | 10 ++ common/src/nexus_config.rs | 27 +++- gateway/Cargo.toml | 2 +- gateway/src/bin/mgs.rs | 122 ++++------------ illumos-utils/Cargo.toml | 1 + illumos-utils/src/lib.rs | 1 + illumos-utils/src/scf.rs | 151 +++++++++++++++++++ nexus/src/app/mod.rs | 15 ++ nexus/src/lib.rs | 41 +++++- nexus/tests/config.test.toml | 1 + openapi/wicketd.json | 18 +++ sled-agent/src/services.rs | 55 ++++++- smf/wicketd/manifest.xml | 30 +++- wicketd/Cargo.toml | 1 + wicketd/src/bin/wicketd.rs | 53 +++++-- wicketd/src/context.rs | 3 + wicketd/src/http_entrypoints.rs | 51 +++++++ wicketd/src/lib.rs | 94 +++++++++++- wicketd/src/nexus_proxy.rs | 178 +++++++++++++++++++++++ wicketd/tests/integration_tests/setup.rs | 2 + 21 files changed, 745 insertions(+), 115 deletions(-) create mode 100644 illumos-utils/src/scf.rs create mode 100644 wicketd/src/nexus_proxy.rs diff --git a/Cargo.lock b/Cargo.lock index 3265ed19de..835fab53cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3418,6 +3418,7 @@ dependencies = [ "byteorder", "camino", "cfg-if 1.0.0", + "crucible-smf", "futures", "ipnetwork", "libc", @@ -4998,7 +4999,6 @@ dependencies = [ "async-trait", "ciborium", "clap 4.4.3", - "crucible-smf", "dropshot", "expectorate", "futures", @@ -5008,6 +5008,7 @@ dependencies = [ "hex", "http", "hyper", + "illumos-utils", "ipcc-key-value", "omicron-common 0.1.0", "omicron-test-utils", @@ -10111,6 +10112,7 @@ dependencies = [ "installinator-artifact-client", "installinator-artifactd", "installinator-common", + "internal-dns 0.1.0", "itertools 0.11.0", "omicron-certificates", "omicron-common 0.1.0", diff --git a/common/src/address.rs b/common/src/address.rs index 0358787258..baa344ef22 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -47,6 +47,16 @@ pub const CRUCIBLE_PANTRY_PORT: u16 = 17000; pub const NEXUS_INTERNAL_PORT: u16 = 12221; +/// The port on which Nexus exposes its external API on the underlay network. +/// +/// This is used by the `wicketd` Nexus proxy to allow external API access via +/// the rack's tech port. +pub const NEXUS_TECHPORT_EXTERNAL_PORT: u16 = 12228; + +/// The port on which `wicketd` runs a Nexus external API proxy on the tech port +/// interface(s). +pub const WICKETD_NEXUS_PROXY_PORT: u16 = 12229; + pub const NTP_PORT: u16 = 123; // The number of ports available to an SNAT IP. diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index ad62c34f92..6b0960643e 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -5,6 +5,7 @@ //! Configuration parameters to Nexus that are usually only known //! at deployment time. +use crate::address::NEXUS_TECHPORT_EXTERNAL_PORT; use crate::api::internal::shared::SwitchLocation; use super::address::{Ipv6Subnet, RACK_PREFIX}; @@ -132,6 +133,19 @@ pub struct DeploymentConfig { pub id: Uuid, /// Uuid of the Rack where Nexus is executing. pub rack_id: Uuid, + /// Port on which the "techport external" dropshot server should listen. + /// This dropshot server copies _most_ of its config from + /// `dropshot_external` (so that it matches TLS, etc.), but builds its + /// listening address by combining `dropshot_internal`'s IP address with + /// this port. + /// + /// We use `serde(default = ...)` to ensure we don't break any serialized + /// configs that were created before this field was added. In production we + /// always expect this port to be constant, but we need to be able to + /// override it when running tests. + #[schemars(skip)] + #[serde(default = "default_techport_external_server_port")] + pub techport_external_server_port: u16, /// Dropshot configuration for the external API server. #[schemars(skip)] // TODO we're protected against dropshot changes pub dropshot_external: ConfigDropshotWithTls, @@ -147,6 +161,10 @@ pub struct DeploymentConfig { pub external_dns_servers: Vec, } +fn default_techport_external_server_port() -> u16 { + NEXUS_TECHPORT_EXTERNAL_PORT +} + impl DeploymentConfig { /// Load a `DeploymentConfig` from the given TOML file /// @@ -442,8 +460,9 @@ impl std::fmt::Display for SchemeName { mod test { use super::Tunables; use super::{ - AuthnConfig, Config, ConsoleConfig, LoadError, PackageConfig, - SchemeName, TimeseriesDbConfig, UpdatesConfig, + default_techport_external_server_port, AuthnConfig, Config, + ConsoleConfig, LoadError, PackageConfig, SchemeName, + TimeseriesDbConfig, UpdatesConfig, }; use crate::address::{Ipv6Subnet, RACK_PREFIX}; use crate::api::internal::shared::SwitchLocation; @@ -611,6 +630,8 @@ mod test { rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f" .parse() .unwrap(), + techport_external_server_port: + default_techport_external_server_port(), dropshot_external: ConfigDropshotWithTls { tls: false, dropshot: ConfigDropshot { @@ -709,6 +730,7 @@ mod test { [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" + techport_external_server_port = 12345 external_dns_servers = [ "1.1.1.1", "9.9.9.9" ] [deployment.dropshot_external] bind_address = "10.1.2.3:4567" @@ -743,6 +765,7 @@ mod test { config.pkg.authn.schemes_external, vec![SchemeName::Spoof, SchemeName::SessionCookie], ); + assert_eq!(config.deployment.techport_external_server_port, 12345); } #[test] diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 07934a6ad3..40899d22a1 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -9,7 +9,6 @@ anyhow.workspace = true async-trait.workspace = true ciborium.workspace = true clap.workspace = true -crucible-smf.workspace = true dropshot.workspace = true futures.workspace = true gateway-messages.workspace = true @@ -17,6 +16,7 @@ gateway-sp-comms.workspace = true hex.workspace = true http.workspace = true hyper.workspace = true +illumos-utils.workspace = true ipcc-key-value.workspace = true omicron-common.workspace = true once_cell.workspace = true diff --git a/gateway/src/bin/mgs.rs b/gateway/src/bin/mgs.rs index cb9070a9a5..81b10ef669 100644 --- a/gateway/src/bin/mgs.rs +++ b/gateway/src/bin/mgs.rs @@ -85,12 +85,11 @@ async fn do_run() -> Result<(), CmdError> { )) })?; - let mut signals = - Signals::new(&[signal::SIGUSR1]).map_err(|e| { - CmdError::Failure(format!( - "failed to set up signal handler: {e}" - )) - })?; + let mut signals = Signals::new([signal::SIGUSR1]).map_err(|e| { + CmdError::Failure(format!( + "failed to set up signal handler: {e}" + )) + })?; let (id, addresses, rack_id) = if id_and_address_from_smf { let config = read_smf_config()?; @@ -141,7 +140,11 @@ async fn do_run() -> Result<(), CmdError> { #[cfg(target_os = "illumos")] fn read_smf_config() -> Result { - use crucible_smf::{Scf, ScfError}; + fn scf_to_cmd_err(err: illumos_utils::scf::ScfError) -> CmdError { + CmdError::Failure(err.to_string()) + } + + use illumos_utils::scf::ScfHandle; // Name of our config property group; must match our SMF manifest.xml. const CONFIG_PG: &str = "config"; @@ -155,107 +158,46 @@ fn read_smf_config() -> Result { // Name of the property within CONFIG_PG for our rack ID. const PROP_RACK_ID: &str = "rack_id"; - // This function is pretty boilerplate-y; we can reduce it by using this - // error type to help us construct a `CmdError::Failure(_)` string. It - // assumes (for the purposes of error messages) any property being fetched - // lives under the `CONFIG_PG` property group. - #[derive(Debug, thiserror::Error)] - enum Error { - #[error("failed to create scf handle: {0}")] - ScfHandle(ScfError), - #[error("failed to get self smf instance: {0}")] - SelfInstance(ScfError), - #[error("failed to get self running snapshot: {0}")] - RunningSnapshot(ScfError), - #[error("failed to get propertygroup `{CONFIG_PG}`: {0}")] - GetPg(ScfError), - #[error("missing propertygroup `{CONFIG_PG}`")] - MissingPg, - #[error("failed to get property `{CONFIG_PG}/{prop}`: {err}")] - GetProperty { prop: &'static str, err: ScfError }, - #[error("missing property `{CONFIG_PG}/{prop}`")] - MissingProperty { prop: &'static str }, - #[error("failed to get value for `{CONFIG_PG}/{prop}`: {err}")] - GetValue { prop: &'static str, err: ScfError }, - #[error("failed to get values for `{CONFIG_PG}/{prop}`: {err}")] - GetValues { prop: &'static str, err: ScfError }, - #[error("failed to get value for `{CONFIG_PG}/{prop}`")] - MissingValue { prop: &'static str }, - #[error("failed to get `{CONFIG_PG}/{prop} as a string: {err}")] - ValueAsString { prop: &'static str, err: ScfError }, - } - - impl From for CmdError { - fn from(err: Error) -> Self { - Self::Failure(err.to_string()) - } - } - - let scf = Scf::new().map_err(Error::ScfHandle)?; - let instance = scf.get_self_instance().map_err(Error::SelfInstance)?; - let snapshot = - instance.get_running_snapshot().map_err(Error::RunningSnapshot)?; - - let config = snapshot - .get_pg("config") - .map_err(Error::GetPg)? - .ok_or(Error::MissingPg)?; + let scf = ScfHandle::new().map_err(scf_to_cmd_err)?; + let instance = scf.self_instance().map_err(scf_to_cmd_err)?; + let snapshot = instance.running_snapshot().map_err(scf_to_cmd_err)?; + let config = snapshot.property_group(CONFIG_PG).map_err(scf_to_cmd_err)?; - let prop_id = config - .get_property(PROP_ID) - .map_err(|err| Error::GetProperty { prop: PROP_ID, err })? - .ok_or_else(|| Error::MissingProperty { prop: PROP_ID })? - .value() - .map_err(|err| Error::GetValue { prop: PROP_ID, err })? - .ok_or(Error::MissingValue { prop: PROP_ID })? - .as_string() - .map_err(|err| Error::ValueAsString { prop: PROP_ID, err })?; + let prop_id = config.value_as_string(PROP_ID).map_err(scf_to_cmd_err)?; let prop_id = Uuid::try_parse(&prop_id).map_err(|err| { CmdError::Failure(format!( - "failed to parse `{CONFIG_PG}/{PROP_ID}` ({prop_id:?}) as a UUID: {err}" + "failed to parse `{CONFIG_PG}/{PROP_ID}` \ + ({prop_id:?}) as a UUID: {err}" )) })?; - let prop_rack_id = config - .get_property(PROP_RACK_ID) - .map_err(|err| Error::GetProperty { prop: PROP_RACK_ID, err })? - .ok_or_else(|| Error::MissingProperty { prop: PROP_RACK_ID })? - .value() - .map_err(|err| Error::GetValue { prop: PROP_RACK_ID, err })? - .ok_or(Error::MissingValue { prop: PROP_RACK_ID })? - .as_string() - .map_err(|err| Error::ValueAsString { prop: PROP_RACK_ID, err })?; + let prop_rack_id = + config.value_as_string(PROP_RACK_ID).map_err(scf_to_cmd_err)?; - let rack_id = if prop_rack_id.as_str() == "unknown" { + let rack_id = if prop_rack_id == "unknown" { None } else { Some(Uuid::try_parse(&prop_rack_id).map_err(|err| { CmdError::Failure(format!( - "failed to parse `{CONFIG_PG}/{PROP_RACK_ID}` ({prop_rack_id:?}) as a UUID: {err}" + "failed to parse `{CONFIG_PG}/{PROP_RACK_ID}` \ + ({prop_rack_id:?}) as a UUID: {err}" )) })?) }; - let prop_addr = config - .get_property(PROP_ADDR) - .map_err(|err| Error::GetProperty { prop: PROP_ADDR, err })? - .ok_or_else(|| Error::MissingProperty { prop: PROP_ADDR })?; + let prop_addr = + config.values_as_strings(PROP_ADDR).map_err(scf_to_cmd_err)?; - let mut addresses = Vec::new(); + let mut addresses = Vec::with_capacity(prop_addr.len()); - for value in prop_addr - .values() - .map_err(|err| Error::GetValues { prop: PROP_ADDR, err })? - { - let addr = value - .map_err(|err| Error::GetValue { prop: PROP_ADDR, err })? - .as_string() - .map_err(|err| Error::ValueAsString { prop: PROP_ADDR, err })?; - - addresses.push(addr.parse().map_err(|err| CmdError::Failure(format!( - "failed to parse `{CONFIG_PG}/{PROP_ADDR}` ({addr:?}) as a socket address: {err}" - )))?); + for addr in prop_addr { + addresses.push(addr.parse().map_err(|err| { + CmdError::Failure(format!( + "failed to parse `{CONFIG_PG}/{PROP_ADDR}` \ + ({addr:?}) as a socket address: {err}" + )) + })?); } if addresses.is_empty() { diff --git a/illumos-utils/Cargo.toml b/illumos-utils/Cargo.toml index e521b54d02..a291a15e78 100644 --- a/illumos-utils/Cargo.toml +++ b/illumos-utils/Cargo.toml @@ -12,6 +12,7 @@ bhyve_api.workspace = true byteorder.workspace = true camino.workspace = true cfg-if.workspace = true +crucible-smf.workspace = true futures.workspace = true ipnetwork.workspace = true libc.workspace = true diff --git a/illumos-utils/src/lib.rs b/illumos-utils/src/lib.rs index 1d585ee786..345f097ae2 100644 --- a/illumos-utils/src/lib.rs +++ b/illumos-utils/src/lib.rs @@ -17,6 +17,7 @@ pub mod libc; pub mod link; pub mod opte; pub mod running_zone; +pub mod scf; pub mod svc; pub mod vmm_reservoir; pub mod zfs; diff --git a/illumos-utils/src/scf.rs b/illumos-utils/src/scf.rs new file mode 100644 index 0000000000..a691146531 --- /dev/null +++ b/illumos-utils/src/scf.rs @@ -0,0 +1,151 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Utilities for accessing SMF properties. + +pub use crucible_smf::ScfError as InnerScfError; + +#[derive(Debug, thiserror::Error)] +pub enum ScfError { + #[error("failed to create scf handle: {0}")] + ScfHandle(InnerScfError), + #[error("failed to get self smf instance: {0}")] + SelfInstance(InnerScfError), + #[error("failed to get self running snapshot: {0}")] + RunningSnapshot(InnerScfError), + #[error("failed to get propertygroup `{group}`: {err}")] + GetPg { group: &'static str, err: InnerScfError }, + #[error("missing propertygroup `{group}`")] + MissingPg { group: &'static str }, + #[error("failed to get property `{group}/{prop}`: {err}")] + GetProperty { group: &'static str, prop: &'static str, err: InnerScfError }, + #[error("missing property `{group}/{prop}`")] + MissingProperty { group: &'static str, prop: &'static str }, + #[error("failed to get value for `{group}/{prop}`: {err}")] + GetValue { group: &'static str, prop: &'static str, err: InnerScfError }, + #[error("failed to get values for `{group}/{prop}`: {err}")] + GetValues { group: &'static str, prop: &'static str, err: InnerScfError }, + #[error("failed to get value for `{group}/{prop}`")] + MissingValue { group: &'static str, prop: &'static str }, + #[error("failed to get `{group}/{prop} as a string: {err}")] + ValueAsString { + group: &'static str, + prop: &'static str, + err: InnerScfError, + }, +} + +pub struct ScfHandle { + inner: crucible_smf::Scf, +} + +impl ScfHandle { + pub fn new() -> Result { + match crucible_smf::Scf::new() { + Ok(inner) => Ok(Self { inner }), + Err(err) => Err(ScfError::ScfHandle(err)), + } + } + + pub fn self_instance(&self) -> Result, ScfError> { + match self.inner.get_self_instance() { + Ok(inner) => Ok(ScfInstance { inner }), + Err(err) => Err(ScfError::SelfInstance(err)), + } + } +} + +pub struct ScfInstance<'a> { + inner: crucible_smf::Instance<'a>, +} + +impl ScfInstance<'_> { + pub fn running_snapshot(&self) -> Result, ScfError> { + match self.inner.get_running_snapshot() { + Ok(inner) => Ok(ScfSnapshot { inner }), + Err(err) => Err(ScfError::RunningSnapshot(err)), + } + } +} + +pub struct ScfSnapshot<'a> { + inner: crucible_smf::Snapshot<'a>, +} + +impl ScfSnapshot<'_> { + pub fn property_group( + &self, + group: &'static str, + ) -> Result, ScfError> { + match self.inner.get_pg(group) { + Ok(Some(inner)) => Ok(ScfPropertyGroup { group, inner }), + Ok(None) => Err(ScfError::MissingPg { group }), + Err(err) => Err(ScfError::GetPg { group, err }), + } + } +} + +pub struct ScfPropertyGroup<'a> { + group: &'static str, + inner: crucible_smf::PropertyGroup<'a>, +} + +impl ScfPropertyGroup<'_> { + fn property<'a>( + &'a self, + prop: &'static str, + ) -> Result, ScfError> { + match self.inner.get_property(prop) { + Ok(Some(prop)) => Ok(prop), + Ok(None) => { + Err(ScfError::MissingProperty { group: self.group, prop }) + } + Err(err) => { + Err(ScfError::GetProperty { group: self.group, prop, err }) + } + } + } + + pub fn value_as_string( + &self, + prop: &'static str, + ) -> Result { + let inner = self.property(prop)?; + let value = inner + .value() + .map_err(|err| ScfError::GetValue { group: self.group, prop, err })? + .ok_or(ScfError::MissingValue { group: self.group, prop })?; + value.as_string().map_err(|err| ScfError::ValueAsString { + group: self.group, + prop, + err, + }) + } + + pub fn values_as_strings( + &self, + prop: &'static str, + ) -> Result, ScfError> { + let inner = self.property(prop)?; + let values = inner.values().map_err(|err| ScfError::GetValues { + group: self.group, + prop, + err, + })?; + values + .map(|value| { + let value = value.map_err(|err| ScfError::GetValue { + group: self.group, + prop, + err, + })?; + value.as_string().map_err(|err| ScfError::ValueAsString { + group: self.group, + prop, + err, + }) + }) + .collect() + } +} diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 354df0ead3..45f69848e3 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -114,6 +114,10 @@ pub struct Nexus { /// External dropshot servers external_server: std::sync::Mutex>, + /// External dropshot server that listens on the internal network to allow + /// connections from the tech port; see RFD 431. + techport_external_server: std::sync::Mutex>, + /// Internal dropshot server internal_server: std::sync::Mutex>, @@ -307,6 +311,7 @@ impl Nexus { sec_client: Arc::clone(&sec_client), recovery_task: std::sync::Mutex::new(None), external_server: std::sync::Mutex::new(None), + techport_external_server: std::sync::Mutex::new(None), internal_server: std::sync::Mutex::new(None), populate_status, timeseries_client, @@ -439,6 +444,7 @@ impl Nexus { pub(crate) async fn set_servers( &self, external_server: DropshotServer, + techport_external_server: DropshotServer, internal_server: DropshotServer, ) { // If any servers already exist, close them. @@ -446,6 +452,10 @@ impl Nexus { // Insert the new servers. self.external_server.lock().unwrap().replace(external_server); + self.techport_external_server + .lock() + .unwrap() + .replace(techport_external_server); self.internal_server.lock().unwrap().replace(internal_server); } @@ -454,6 +464,11 @@ impl Nexus { if let Some(server) = external_server { server.close().await?; } + let techport_external_server = + self.techport_external_server.lock().unwrap().take(); + if let Some(server) = techport_external_server { + server.close().await?; + } let internal_server = self.internal_server.lock().unwrap().take(); if let Some(server) = internal_server { server.close().await?; diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index 586c828683..0ada48e203 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -26,6 +26,7 @@ pub use app::test_interfaces::TestInterfaces; pub use app::Nexus; pub use config::Config; use context::ServerContext; +use dropshot::ConfigDropshot; use external_api::http_entrypoints::external_api; use internal_api::http_entrypoints::internal_api; use nexus_types::internal_api::params::ServiceKind; @@ -131,6 +132,23 @@ impl Server { .nexus .external_tls_config(config.deployment.dropshot_external.tls) .await; + + // We launch two dropshot servers providing the external API: one as + // configured (which is accessible from the customer network), and one + // that matches the configuration except listens on the same address + // (but a different port) as the `internal` server. The latter is + // available for proxied connections via the tech port in the event the + // rack has lost connectivity (see RFD 431). + let techport_server_bind_addr = { + let mut addr = http_server_internal.local_addr(); + addr.set_port(config.deployment.techport_external_server_port); + addr + }; + let techport_server_config = ConfigDropshot { + bind_address: techport_server_bind_addr, + ..config.deployment.dropshot_external.dropshot.clone() + }; + let http_server_external = { let server_starter_external = dropshot::HttpServerStarter::new_with_tls( @@ -138,16 +156,35 @@ impl Server { external_api(), Arc::clone(&apictx), &log.new(o!("component" => "dropshot_external")), - tls_config.map(dropshot::ConfigTls::Dynamic), + tls_config.clone().map(dropshot::ConfigTls::Dynamic), ) .map_err(|error| { format!("initializing external server: {}", error) })?; server_starter_external.start() }; + let http_server_techport_external = { + let server_starter_external_techport = + dropshot::HttpServerStarter::new_with_tls( + &techport_server_config, + external_api(), + Arc::clone(&apictx), + &log.new(o!("component" => "dropshot_external_techport")), + tls_config.map(dropshot::ConfigTls::Dynamic), + ) + .map_err(|error| { + format!("initializing external techport server: {}", error) + })?; + server_starter_external_techport.start() + }; + apictx .nexus - .set_servers(http_server_external, http_server_internal) + .set_servers( + http_server_external, + http_server_techport_external, + http_server_internal, + ) .await; let server = Server { apictx: apictx.clone() }; Ok(server) diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 1b1ae2c912..09f13e55c7 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -39,6 +39,7 @@ max_vpc_ipv4_subnet_prefix = 29 # NOTE: The test suite always overrides this. id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc" +techport_external_server_port = 0 # Nexus may need to resolve external hosts (e.g. to grab IdP metadata). # These are the DNS servers it should use. diff --git a/openapi/wicketd.json b/openapi/wicketd.json index d67fc79f7a..8b4da8970f 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -569,6 +569,24 @@ } } }, + "/reload-config": { + "post": { + "summary": "An endpoint instructing wicketd to reload its SMF config properties.", + "description": "The only expected client of this endpoint is `curl` from wicketd's SMF `refresh` method, but other clients hitting it is harmless.", + "operationId": "post_reload_config", + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/repository": { "put": { "summary": "Upload a TUF repository to the server.", diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 60f0965612..f91b5091e6 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -62,7 +62,6 @@ use illumos_utils::zone::Zones; use illumos_utils::{execute, PFEXEC}; use internal_dns::resolver::Resolver; use itertools::Itertools; -use omicron_common::address::Ipv6Subnet; use omicron_common::address::AZ_PREFIX; use omicron_common::address::BOOTSTRAP_ARTIFACT_PORT; use omicron_common::address::CLICKHOUSE_KEEPER_PORT; @@ -74,7 +73,9 @@ use omicron_common::address::DENDRITE_PORT; use omicron_common::address::MGS_PORT; use omicron_common::address::RACK_PREFIX; use omicron_common::address::SLED_PREFIX; +use omicron_common::address::WICKETD_NEXUS_PROXY_PORT; use omicron_common::address::WICKETD_PORT; +use omicron_common::address::{Ipv6Subnet, NEXUS_TECHPORT_EXTERNAL_PORT}; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::RackNetworkConfig; use omicron_common::backoff::{ @@ -1448,6 +1449,8 @@ impl ServiceManager { let deployment_config = NexusDeploymentConfig { id: request.zone.id, rack_id: sled_info.rack_id, + techport_external_server_port: + NEXUS_TECHPORT_EXTERNAL_PORT, dropshot_external: ConfigDropshotWithTls { tls: *external_tls, @@ -1696,6 +1699,28 @@ impl ServiceManager { &format!("[::1]:{MGS_PORT}"), )?; + // We intentionally bind `nexus-proxy-address` to `::` so + // wicketd will serve this on all interfaces, particularly + // the tech port interfaces, allowing external clients to + // connect to this Nexus proxy. + smfh.setprop( + "config/nexus-proxy-address", + &format!("[::]:{WICKETD_NEXUS_PROXY_PORT}"), + )?; + if let Some(underlay_address) = self + .inner + .sled_info + .get() + .map(|info| info.underlay_address) + { + let rack_subnet = + Ipv6Subnet::::new(underlay_address); + smfh.setprop( + "config/rack-subnet", + &rack_subnet.net().ip().to_string(), + )?; + } + let serialized_baseboard = serde_json::to_string_pretty(&baseboard)?; let serialized_baseboard_path = Utf8PathBuf::from(format!( @@ -2705,9 +2730,8 @@ impl ServiceManager { ); *request = new_request; - let address = request - .addresses - .get(0) + let first_address = request.addresses.get(0); + let address = first_address .map(|addr| addr.to_string()) .unwrap_or_else(|| "".to_string()); @@ -2813,6 +2837,29 @@ impl ServiceManager { } smfh.refresh()?; } + ServiceType::Wicketd { .. } => { + if let Some(&address) = first_address { + let rack_subnet = + Ipv6Subnet::::new(address); + + info!( + self.inner.log, "configuring wicketd"; + "rack_subnet" => %rack_subnet.net().ip(), + ); + + smfh.setprop( + "config/rack-subnet", + &rack_subnet.net().ip().to_string(), + )?; + + smfh.refresh()?; + } else { + error!( + self.inner.log, + "underlay address unexpectedly missing", + ); + } + } ServiceType::Tfport { .. } => { // Since tfport and dpd communicate using localhost, // the tfport service shouldn't need to be restarted. diff --git a/smf/wicketd/manifest.xml b/smf/wicketd/manifest.xml index cb7ed657e0..778a7abf2d 100644 --- a/smf/wicketd/manifest.xml +++ b/smf/wicketd/manifest.xml @@ -12,10 +12,29 @@ + + + @@ -24,7 +43,16 @@ + + + + diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index 1044e1ff51..cf04b7c6a7 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -24,6 +24,7 @@ hubtools.workspace = true http.workspace = true hyper.workspace = true illumos-utils.workspace = true +internal-dns.workspace = true itertools.workspace = true reqwest.workspace = true schemars.workspace = true diff --git a/wicketd/src/bin/wicketd.rs b/wicketd/src/bin/wicketd.rs index adfac5ac1a..2e6d51c0f0 100644 --- a/wicketd/src/bin/wicketd.rs +++ b/wicketd/src/bin/wicketd.rs @@ -5,11 +5,14 @@ //! Executable for wicketd: technician port based management service use clap::Parser; -use omicron_common::cmd::{fatal, CmdError}; +use omicron_common::{ + address::Ipv6Subnet, + cmd::{fatal, CmdError}, +}; use sled_hardware::Baseboard; -use std::net::SocketAddrV6; +use std::net::{Ipv6Addr, SocketAddrV6}; use std::path::PathBuf; -use wicketd::{self, run_openapi, Config, Server}; +use wicketd::{self, run_openapi, Config, Server, SmfConfigValues}; #[derive(Debug, Parser)] #[clap(name = "wicketd", about = "See README.adoc for more information")] @@ -30,12 +33,28 @@ enum Args { #[clap(long, action)] artifact_address: SocketAddrV6, - /// The port on localhost for MGS + /// The address (expected to be on localhost) for MGS #[clap(long, action)] mgs_address: SocketAddrV6, + /// The address (expected to be on localhost) on which we'll serve a TCP + /// proxy to Nexus's "techport external" API + #[clap(long, action)] + nexus_proxy_address: SocketAddrV6, + + /// Path to a file containing our baseboard information #[clap(long)] baseboard_file: Option, + + /// Read dynamic properties from our SMF config instead of passing them + /// on the command line + #[clap(long)] + read_smf_config: bool, + + /// The subnet for the rack; typically read directly from our SMF config + /// via `--read-smf-config` or an SMF refresh + #[clap(long, action, conflicts_with("read_smf_config"))] + rack_subnet: Option, }, } @@ -56,20 +75,22 @@ async fn do_run() -> Result<(), CmdError> { address, artifact_address, mgs_address, + nexus_proxy_address, baseboard_file, + read_smf_config, + rack_subnet, } => { let baseboard = if let Some(baseboard_file) = baseboard_file { - let baseboard_file = - std::fs::read_to_string(&baseboard_file) - .map_err(|e| CmdError::Failure(e.to_string()))?; + let baseboard_file = std::fs::read_to_string(baseboard_file) + .map_err(|e| CmdError::Failure(e.to_string()))?; let baseboard: Baseboard = serde_json::from_str(&baseboard_file) .map_err(|e| CmdError::Failure(e.to_string()))?; // TODO-correctness `Baseboard::unknown()` is slated for removal - // after some refactoring in sled-agent, at which point we'll need a - // different way for sled-agent to tell us it doesn't know our - // baseboard. + // after some refactoring in sled-agent, at which point we'll + // need a different way for sled-agent to tell us it doesn't + // know our baseboard. if matches!(baseboard, Baseboard::Unknown) { None } else { @@ -87,11 +108,23 @@ async fn do_run() -> Result<(), CmdError> { )) })?; + let rack_subnet = match rack_subnet { + Some(addr) => Some(Ipv6Subnet::new(addr)), + None if read_smf_config => { + let smf_values = SmfConfigValues::read_current() + .map_err(|e| CmdError::Failure(e.to_string()))?; + smf_values.rack_subnet + } + None => None, + }; + let args = wicketd::Args { address, artifact_address, mgs_address, + nexus_proxy_address, baseboard, + rack_subnet, }; let log = config.log.to_logger("wicketd").map_err(|msg| { CmdError::Failure(format!("initializing logger: {}", msg)) diff --git a/wicketd/src/context.rs b/wicketd/src/context.rs index db1c72fcb9..eeecc3fa64 100644 --- a/wicketd/src/context.rs +++ b/wicketd/src/context.rs @@ -13,6 +13,7 @@ use anyhow::anyhow; use anyhow::bail; use anyhow::Result; use gateway_client::types::SpIdentifier; +use internal_dns::resolver::Resolver; use sled_hardware::Baseboard; use slog::info; use std::net::Ipv6Addr; @@ -23,6 +24,7 @@ use std::sync::OnceLock; /// Shared state used by API handlers pub struct ServerContext { + pub(crate) bind_address: SocketAddrV6, pub mgs_handle: MgsHandle, pub mgs_client: gateway_client::Client, pub(crate) log: slog::Logger, @@ -36,6 +38,7 @@ pub struct ServerContext { pub(crate) baseboard: Option, pub(crate) rss_config: Mutex, pub(crate) preflight_checker: PreflightCheckerHandler, + pub(crate) internal_dns_resolver: Arc>>, } impl ServerContext { diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs index 72c3341334..be0f681601 100644 --- a/wicketd/src/http_entrypoints.rs +++ b/wicketd/src/http_entrypoints.rs @@ -12,6 +12,7 @@ use crate::mgs::MgsHandle; use crate::mgs::ShutdownInProgress; use crate::preflight_check::UplinkEventReport; use crate::RackV1Inventory; +use crate::SmfConfigValues; use bootstrap_agent_client::types::RackInitId; use bootstrap_agent_client::types::RackOperationStatus; use bootstrap_agent_client::types::RackResetId; @@ -29,6 +30,7 @@ use gateway_client::types::IgnitionCommand; use gateway_client::types::SpIdentifier; use gateway_client::types::SpType; use http::StatusCode; +use internal_dns::resolver::Resolver; use omicron_common::address; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::shared::RackNetworkConfig; @@ -39,6 +41,7 @@ use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; use sled_hardware::Baseboard; +use slog::o; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::io; @@ -80,6 +83,7 @@ pub fn api() -> WicketdApiDescription { api.register(post_ignition_command)?; api.register(post_start_preflight_uplink_check)?; api.register(get_preflight_uplink_report)?; + api.register(post_reload_config)?; Ok(()) } @@ -1237,6 +1241,53 @@ async fn get_preflight_uplink_report( } } +/// An endpoint instructing wicketd to reload its SMF config properties. +/// +/// The only expected client of this endpoint is `curl` from wicketd's SMF +/// `refresh` method, but other clients hitting it is harmless. +#[endpoint { + method = POST, + path = "/reload-config", +}] +async fn post_reload_config( + rqctx: RequestContext, +) -> Result { + let smf_values = SmfConfigValues::read_current().map_err(|err| { + HttpError::for_unavail( + None, + format!("failed to read SMF values: {err}"), + ) + })?; + + let rqctx = rqctx.context(); + + // We do not allow a config reload to change our bound address; return an + // error if the caller is attempting to do so. + if rqctx.bind_address != smf_values.address { + return Err(HttpError::for_bad_request( + None, + "listening address cannot be reconfigured".to_string(), + )); + } + + if let Some(rack_subnet) = smf_values.rack_subnet { + let resolver = Resolver::new_from_subnet( + rqctx.log.new(o!("component" => "InternalDnsResolver")), + rack_subnet, + ) + .map_err(|err| { + HttpError::for_unavail( + None, + format!("failed to create internal DNS resolver: {err}"), + ) + })?; + + *rqctx.internal_dns_resolver.lock().unwrap() = Some(resolver); + } + + Ok(HttpResponseUpdatedNoContent()) +} + fn http_error_from_client_error( err: gateway_client::Error, ) -> HttpError { diff --git a/wicketd/src/lib.rs b/wicketd/src/lib.rs index e17c15642c..ada1902654 100644 --- a/wicketd/src/lib.rs +++ b/wicketd/src/lib.rs @@ -11,6 +11,7 @@ mod http_entrypoints; mod installinator_progress; mod inventory; pub mod mgs; +mod nexus_proxy; mod preflight_check; mod rss_config; mod update_tracker; @@ -22,14 +23,17 @@ pub use config::Config; pub(crate) use context::ServerContext; use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer}; pub use installinator_progress::{IprUpdateTracker, RunningUpdateState}; +use internal_dns::resolver::Resolver; pub use inventory::{RackV1Inventory, SpInventory}; use mgs::make_mgs_client; pub(crate) use mgs::{MgsHandle, MgsManager}; +use nexus_proxy::NexusTcpProxy; +use omicron_common::address::{Ipv6Subnet, AZ_PREFIX}; use omicron_common::FileKv; use preflight_check::PreflightCheckerHandler; use sled_hardware::Baseboard; use slog::{debug, error, o, Drain}; -use std::sync::OnceLock; +use std::sync::{Mutex, OnceLock}; use std::{ net::{SocketAddr, SocketAddrV6}, sync::Arc, @@ -53,7 +57,62 @@ pub struct Args { pub address: SocketAddrV6, pub artifact_address: SocketAddrV6, pub mgs_address: SocketAddrV6, + pub nexus_proxy_address: SocketAddrV6, pub baseboard: Option, + pub rack_subnet: Option>, +} + +pub struct SmfConfigValues { + pub address: SocketAddrV6, + pub rack_subnet: Option>, +} + +impl SmfConfigValues { + #[cfg(target_os = "illumos")] + pub fn read_current() -> Result { + use anyhow::Context; + use illumos_utils::scf::ScfHandle; + + const CONFIG_PG: &str = "config"; + const PROP_RACK_SUBNET: &str = "rack-subnet"; + const PROP_ADDRESS: &str = "address"; + + let scf = ScfHandle::new()?; + let instance = scf.self_instance()?; + let snapshot = instance.running_snapshot()?; + let config = snapshot.property_group(CONFIG_PG)?; + + let rack_subnet = config.value_as_string(PROP_RACK_SUBNET)?; + + let rack_subnet = if rack_subnet == "unknown" { + None + } else { + let addr = rack_subnet.parse().with_context(|| { + format!( + "failed to parse {CONFIG_PG}/{PROP_RACK_SUBNET} \ + value {rack_subnet:?} as an IP address" + ) + })?; + Some(Ipv6Subnet::new(addr)) + }; + + let address = { + let address = config.value_as_string(PROP_ADDRESS)?; + address.parse().with_context(|| { + format!( + "failed to parse {CONFIG_PG}/{PROP_ADDRESS} \ + value {address:?} as a socket address" + ) + })? + }; + + Ok(Self { address, rack_subnet }) + } + + #[cfg(not(target_os = "illumos"))] + pub fn read_current() -> Result { + Err(anyhow!("reading SMF config only available on illumos")) + } } pub struct Server { @@ -62,6 +121,7 @@ pub struct Server { pub artifact_store: WicketdArtifactStore, pub update_tracker: Arc, pub ipr_update_tracker: IprUpdateTracker, + nexus_tcp_proxy: NexusTcpProxy, } impl Server { @@ -80,8 +140,9 @@ impl Server { let dropshot_config = ConfigDropshot { bind_address: SocketAddr::V6(args.address), - // The maximum request size is set to 4 GB -- artifacts can be large and there's currently - // no way to set a larger request size for some endpoints. + // The maximum request size is set to 4 GB -- artifacts can be large + // and there's currently no way to set a larger request size for + // some endpoints. request_body_max_bytes: 4 << 30, default_handler_task_mode: HandlerTaskMode::Detached, }; @@ -104,6 +165,27 @@ impl Server { )); let bootstrap_peers = BootstrapPeers::new(&log); + let internal_dns_resolver = args + .rack_subnet + .map(|addr| { + Resolver::new_from_subnet( + log.new(o!("component" => "InternalDnsResolver")), + addr, + ) + .map_err(|err| { + format!("Could not create internal DNS resolver: {err}") + }) + }) + .transpose()?; + + let internal_dns_resolver = Arc::new(Mutex::new(internal_dns_resolver)); + let nexus_tcp_proxy = NexusTcpProxy::start( + args.nexus_proxy_address, + Arc::clone(&internal_dns_resolver), + &log, + ) + .await + .map_err(|err| format!("failed to start Nexus TCP proxy: {err}"))?; let wicketd_server = { let ds_log = log.new(o!("component" => "dropshot (wicketd)")); @@ -112,6 +194,7 @@ impl Server { &dropshot_config, http_entrypoints::api(), ServerContext { + bind_address: args.address, mgs_handle, mgs_client, log: log.clone(), @@ -121,6 +204,7 @@ impl Server { baseboard: args.baseboard, rss_config: Default::default(), preflight_checker: PreflightCheckerHandler::new(&log), + internal_dns_resolver, }, &ds_log, ) @@ -146,17 +230,19 @@ impl Server { artifact_store: store, update_tracker, ipr_update_tracker, + nexus_tcp_proxy, }) } /// Close all running dropshot servers. - pub async fn close(self) -> Result<()> { + pub async fn close(mut self) -> Result<()> { self.wicketd_server.close().await.map_err(|error| { anyhow!("error closing wicketd server: {error}") })?; self.artifact_server.close().await.map_err(|error| { anyhow!("error closing artifact server: {error}") })?; + self.nexus_tcp_proxy.shutdown(); Ok(()) } diff --git a/wicketd/src/nexus_proxy.rs b/wicketd/src/nexus_proxy.rs new file mode 100644 index 0000000000..33ff02a945 --- /dev/null +++ b/wicketd/src/nexus_proxy.rs @@ -0,0 +1,178 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! TCP proxy to expose Nexus's external API via the techport. + +use internal_dns::resolver::Resolver; +use internal_dns::ServiceName; +use omicron_common::address::NEXUS_TECHPORT_EXTERNAL_PORT; +use slog::info; +use slog::o; +use slog::warn; +use slog::Logger; +use std::io; +use std::net::SocketAddr; +use std::net::SocketAddrV6; +use std::sync::Arc; +use std::sync::Mutex; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio::sync::oneshot; + +pub(crate) struct NexusTcpProxy { + shutdown_tx: Option>, +} + +impl Drop for NexusTcpProxy { + fn drop(&mut self) { + self.shutdown(); + } +} + +impl NexusTcpProxy { + pub(crate) async fn start( + listen_addr: SocketAddrV6, + internal_dns_resolver: Arc>>, + log: &Logger, + ) -> io::Result { + let listener = TcpListener::bind(listen_addr).await?; + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + + let inner = Inner { + listener, + internal_dns_resolver, + shutdown_rx, + log: log.new(o!("component" => "NexusTcpProxy")), + }; + + tokio::spawn(inner.run()); + + Ok(Self { shutdown_tx: Some(shutdown_tx) }) + } + + pub(crate) fn shutdown(&mut self) { + if let Some(tx) = self.shutdown_tx.take() { + // We want to shutdown the task we spawned; failure to message it + // means it's already done. + _ = tx.send(()); + } + } +} + +struct Inner { + listener: TcpListener, + internal_dns_resolver: Arc>>, + shutdown_rx: oneshot::Receiver<()>, + log: Logger, +} + +impl Inner { + async fn run(mut self) { + loop { + tokio::select! { + // Cancel-safe per the docs on `TcpListener::accept()` + result = self.listener.accept() => { + self.spawn_proxy_handler(result); + } + + // Cancel-safe: awaiting a `&mut Fut` does not drop the future + _ = &mut self.shutdown_rx => { + info!(self.log, "exiting"); + return; + } + } + } + } + + fn spawn_proxy_handler( + &mut self, + result: io::Result<(TcpStream, SocketAddr)>, + ) { + let (stream, log) = match result { + Ok((stream, peer)) => (stream, self.log.new(o!("peer" => peer))), + Err(err) => { + warn!(self.log, "accept() failed"; "err" => %err); + return; + } + }; + + info!(log, "accepted connection"); + + let Some(resolver) = self.internal_dns_resolver.lock().unwrap().clone() + else { + info!( + log, + "closing connection; no internal DNS resolver available \ + (rack subnet unknown?)" + ); + return; + }; + + tokio::spawn(run_proxy(stream, resolver, log)); + } +} + +async fn run_proxy( + mut client_stream: TcpStream, + resolver: Resolver, + log: Logger, +) { + // Can we talk to the internal DNS server(s) to find Nexus's IPs? + let nexus_addrs = match resolver.lookup_all_ipv6(ServiceName::Nexus).await { + Ok(ips) => ips + .into_iter() + .map(|ip| { + SocketAddr::V6(SocketAddrV6::new( + ip, + NEXUS_TECHPORT_EXTERNAL_PORT, + 0, + 0, + )) + }) + .collect::>(), + Err(err) => { + warn!( + log, "failed to look up Nexus IP addrs"; + "err" => %err, + ); + return; + } + }; + + // Can we connect to any Nexus instance? + let mut nexus_stream = + match TcpStream::connect(nexus_addrs.as_slice()).await { + Ok(stream) => stream, + Err(err) => { + warn!( + log, "failed to connect to Nexus"; + "nexus_addrs" => ?nexus_addrs, + "err" => %err, + ); + return; + } + }; + + let log = match nexus_stream.peer_addr() { + Ok(addr) => log.new(o!("nexus_addr" => addr)), + Err(err) => log.new(o!("nexus_addr" => + format!("failed to read Nexus peer addr: {err}"))), + }; + info!(log, "connected to Nexus"); + + match tokio::io::copy_bidirectional(&mut client_stream, &mut nexus_stream) + .await + { + Ok((client_to_nexus, nexus_to_client)) => { + info!( + log, "closing successful proxy connection to Nexus"; + "bytes_sent_to_nexus" => client_to_nexus, + "bytes_sent_to_client" => nexus_to_client, + ); + } + Err(err) => { + warn!(log, "error proxying data to Nexus"; "err" => %err); + } + } +} diff --git a/wicketd/tests/integration_tests/setup.rs b/wicketd/tests/integration_tests/setup.rs index f0ca183c6a..62682a73ab 100644 --- a/wicketd/tests/integration_tests/setup.rs +++ b/wicketd/tests/integration_tests/setup.rs @@ -40,7 +40,9 @@ impl WicketdTestContext { address: localhost_port_0, artifact_address: localhost_port_0, mgs_address, + nexus_proxy_address: localhost_port_0, baseboard: None, + rack_subnet: None, }; let server = wicketd::Server::start(log.clone(), args) From 463cc1ae256a5e22b9726ae58ea8b651e81e3882 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 18 Oct 2023 09:11:31 -0700 Subject: [PATCH 62/89] Make dbinit.sql slightly less order-dependent (#4288) This PR does two things: 1. It avoids checking for the order-dependent NOT NULL constraint in information schema, opting to parse the output of `SHOW CONSTRAINTS` and [information_schema.columna.is_nullable](https://www.cockroachlabs.com/docs/v23.1/information-schema#columns) instead. 2. It re-orders the db_metadata creation to the end of dbinit.sql (mostly for clarity) Fixes https://github.com/oxidecomputer/omicron/issues/4286 --- nexus/tests/integration_tests/schema.rs | 175 ++++++++++++++++-------- schema/crdb/README.adoc | 41 ++---- schema/crdb/dbinit.sql | 60 ++++---- 3 files changed, 153 insertions(+), 123 deletions(-) diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 6d2595b561..f7d6c1da6a 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -253,7 +253,35 @@ impl<'a> From<&'a [&'static str]> for ColumnSelector<'a> { } } -async fn query_crdb_for_rows_of_strings( +async fn crdb_show_constraints( + crdb: &CockroachInstance, + table: &str, +) -> Vec { + let client = crdb.connect().await.expect("failed to connect"); + + let sql = format!("SHOW CONSTRAINTS FROM {table}"); + let rows = client + .query(&sql, &[]) + .await + .unwrap_or_else(|_| panic!("failed to query {table}")); + client.cleanup().await.expect("cleaning up after wipe"); + + let mut result = vec![]; + for row in rows { + let mut row_result = Row::new(); + for i in 0..row.len() { + let column_name = row.columns()[i].name(); + row_result.values.push(NamedSqlValue { + column: column_name.to_string(), + value: row.get(i), + }); + } + result.push(row_result); + } + result +} + +async fn crdb_select( crdb: &CockroachInstance, columns: ColumnSelector<'_>, table: &str, @@ -453,22 +481,16 @@ async fn versions_have_idempotent_up() { logctx.cleanup_successful(); } -const COLUMNS: [&'static str; 6] = [ +const COLUMNS: [&'static str; 7] = [ "table_catalog", "table_schema", "table_name", "column_name", "column_default", + "is_nullable", "data_type", ]; -const CHECK_CONSTRAINTS: [&'static str; 4] = [ - "constraint_catalog", - "constraint_schema", - "constraint_name", - "check_clause", -]; - const CONSTRAINT_COLUMN_USAGE: [&'static str; 7] = [ "table_catalog", "table_schema", @@ -538,22 +560,9 @@ const PG_INDEXES: [&'static str; 5] = const TABLES: [&'static str; 4] = ["table_catalog", "table_schema", "table_name", "table_type"]; -const TABLE_CONSTRAINTS: [&'static str; 9] = [ - "constraint_catalog", - "constraint_schema", - "constraint_name", - "table_catalog", - "table_schema", - "table_name", - "constraint_type", - "is_deferrable", - "initially_deferred", -]; - #[derive(Eq, PartialEq, Debug)] struct InformationSchema { columns: Vec, - check_constraints: Vec, constraint_column_usage: Vec, key_column_usage: Vec, referential_constraints: Vec, @@ -562,7 +571,7 @@ struct InformationSchema { sequences: Vec, pg_indexes: Vec, tables: Vec, - table_constraints: Vec, + table_constraints: BTreeMap>, } impl InformationSchema { @@ -576,10 +585,6 @@ impl InformationSchema { self.table_constraints, other.table_constraints ); - similar_asserts::assert_eq!( - self.check_constraints, - other.check_constraints - ); similar_asserts::assert_eq!( self.constraint_column_usage, other.constraint_column_usage @@ -602,7 +607,7 @@ impl InformationSchema { // https://www.cockroachlabs.com/docs/v23.1/information-schema // // For details on each of these tables. - let columns = query_crdb_for_rows_of_strings( + let columns = crdb_select( crdb, COLUMNS.as_slice().into(), "information_schema.columns", @@ -610,15 +615,7 @@ impl InformationSchema { ) .await; - let check_constraints = query_crdb_for_rows_of_strings( - crdb, - CHECK_CONSTRAINTS.as_slice().into(), - "information_schema.check_constraints", - None, - ) - .await; - - let constraint_column_usage = query_crdb_for_rows_of_strings( + let constraint_column_usage = crdb_select( crdb, CONSTRAINT_COLUMN_USAGE.as_slice().into(), "information_schema.constraint_column_usage", @@ -626,7 +623,7 @@ impl InformationSchema { ) .await; - let key_column_usage = query_crdb_for_rows_of_strings( + let key_column_usage = crdb_select( crdb, KEY_COLUMN_USAGE.as_slice().into(), "information_schema.key_column_usage", @@ -634,7 +631,7 @@ impl InformationSchema { ) .await; - let referential_constraints = query_crdb_for_rows_of_strings( + let referential_constraints = crdb_select( crdb, REFERENTIAL_CONSTRAINTS.as_slice().into(), "information_schema.referential_constraints", @@ -642,7 +639,7 @@ impl InformationSchema { ) .await; - let views = query_crdb_for_rows_of_strings( + let views = crdb_select( crdb, VIEWS.as_slice().into(), "information_schema.views", @@ -650,7 +647,7 @@ impl InformationSchema { ) .await; - let statistics = query_crdb_for_rows_of_strings( + let statistics = crdb_select( crdb, STATISTICS.as_slice().into(), "information_schema.statistics", @@ -658,7 +655,7 @@ impl InformationSchema { ) .await; - let sequences = query_crdb_for_rows_of_strings( + let sequences = crdb_select( crdb, SEQUENCES.as_slice().into(), "information_schema.sequences", @@ -666,7 +663,7 @@ impl InformationSchema { ) .await; - let pg_indexes = query_crdb_for_rows_of_strings( + let pg_indexes = crdb_select( crdb, PG_INDEXES.as_slice().into(), "pg_indexes", @@ -674,7 +671,7 @@ impl InformationSchema { ) .await; - let tables = query_crdb_for_rows_of_strings( + let tables = crdb_select( crdb, TABLES.as_slice().into(), "information_schema.tables", @@ -682,17 +679,11 @@ impl InformationSchema { ) .await; - let table_constraints = query_crdb_for_rows_of_strings( - crdb, - TABLE_CONSTRAINTS.as_slice().into(), - "information_schema.table_constraints", - Some("table_schema = 'public'"), - ) - .await; + let table_constraints = + Self::show_constraints_all_tables(&tables, crdb).await; Self { columns, - check_constraints, constraint_column_usage, key_column_usage, referential_constraints, @@ -705,6 +696,33 @@ impl InformationSchema { } } + async fn show_constraints_all_tables( + tables: &Vec, + crdb: &CockroachInstance, + ) -> BTreeMap> { + let mut map = BTreeMap::new(); + + for table in tables { + let table = &table.values; + let table_catalog = + table[0].expect("table_catalog").unwrap().as_str(); + let table_schema = + table[1].expect("table_schema").unwrap().as_str(); + let table_name = table[2].expect("table_name").unwrap().as_str(); + let table_type = table[3].expect("table_type").unwrap().as_str(); + + if table_type != "BASE TABLE" { + continue; + } + + let table_name = + format!("{}.{}.{}", table_catalog, table_schema, table_name); + let rows = crdb_show_constraints(crdb, &table_name).await; + map.insert(table_name, rows); + } + map + } + // This would normally be quite an expensive operation, but we expect it'll // at least be slightly cheaper for the freshly populated DB, which // shouldn't have that many records yet. @@ -731,13 +749,9 @@ impl InformationSchema { let table_name = format!("{}.{}.{}", table_catalog, table_schema, table_name); info!(log, "Querying table: {table_name}"); - let rows = query_crdb_for_rows_of_strings( - crdb, - ColumnSelector::Star, - &table_name, - None, - ) - .await; + let rows = + crdb_select(crdb, ColumnSelector::Star, &table_name, None) + .await; info!(log, "Saw data: {rows:?}"); map.insert(table_name, rows); } @@ -1012,6 +1026,47 @@ async fn compare_table_differing_constraint() { ) .await; - assert_ne!(schema1.check_constraints, schema2.check_constraints); + assert_ne!(schema1.table_constraints, schema2.table_constraints); + logctx.cleanup_successful(); +} + +#[tokio::test] +async fn compare_table_differing_not_null_order() { + let config = load_test_config(); + let logctx = LogContext::new( + "compare_table_differing_not_null_order", + &config.pkg.log, + ); + let log = &logctx.log; + + let schema1 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.pet ( id UUID PRIMARY KEY ); + CREATE TABLE omicron.public.employee ( + id UUID PRIMARY KEY, + name TEXT NOT NULL, + hobbies TEXT + ); + ", + ) + .await; + + let schema2 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.employee ( + id UUID PRIMARY KEY, + name TEXT NOT NULL, + hobbies TEXT + ); + CREATE TABLE omicron.public.pet ( id UUID PRIMARY KEY ); + ", + ) + .await; + + schema1.pretty_assert_eq(&schema2); logctx.cleanup_successful(); } diff --git a/schema/crdb/README.adoc b/schema/crdb/README.adoc index c15b51e374..f92748f101 100644 --- a/schema/crdb/README.adoc +++ b/schema/crdb/README.adoc @@ -73,38 +73,15 @@ SQL Validation, via Automated Tests: ==== Handling common schema changes -CockroachDB's schema includes a description of all of the database's CHECK -constraints. If a CHECK constraint is anonymous (i.e. it is written simply as -`CHECK ` and not `CONSTRAINT CHECK expression`), CRDB -assigns it a name based on the table and column to which the constraint applies. -The challenge is that CRDB identifies tables and columns using opaque -identifiers whose values depend on the order in which tables and views were -defined in the current database. This means that adding, removing, or renaming -objects needs to be done carefully to preserve the relative ordering of objects -in new databases created by `dbinit.sql` and upgraded databases created by -applying `up.sql` transformations. - -===== Adding new columns with constraints - -Strongly consider naming new constraints (`CONSTRAINT `) to -avoid the problems with anonymous constraints described above. - -===== Adding tables and views - -New tables and views must be added to the end of `dbinit.sql` so that the order -of preceding `CREATE` statements is left unchanged. If your changes fail the -`CHECK` constraints test and you get a constraint name diff like this... - -``` -NamedSqlValue { - column: "constraint_name", - value: Some( - String( -< "4101115737_149_10_not_null", -> "4101115737_148_10_not_null", -``` - -...then you've probably inadvertently added a table or view in the wrong place. +Although CockroachDB's schema includes some opaque internally-generated fields +that are order dependent - such as the names of anonymous CHECK constraints - +our schema comparison tools intentionally ignore these values. As a result, +when performing schema changes, the order of new tables and constraints should +generally not be important. + +As convention, however, we recommend keeping the `db_metadata` file at the end of +`dbinit.sql`, so that the database does not contain a version until it is fully +populated. ==== Adding new source tables to an existing view diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 9f5f78326c..4d0589b3a0 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -2513,37 +2513,6 @@ BEGIN; * nothing to ensure it gets bumped when it should be, but it's a start. */ -CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( - -- There should only be one row of this table for the whole DB. - -- It's a little goofy, but filter on "singleton = true" before querying - -- or applying updates, and you'll access the singleton row. - -- - -- We also add a constraint on this table to ensure it's not possible to - -- access the version of this table with "singleton = false". - singleton BOOL NOT NULL PRIMARY KEY, - time_created TIMESTAMPTZ NOT NULL, - time_modified TIMESTAMPTZ NOT NULL, - -- Semver representation of the DB version - version STRING(64) NOT NULL, - - -- (Optional) Semver representation of the DB version to which we're upgrading - target_version STRING(64), - - CHECK (singleton = true) -); - -INSERT INTO omicron.public.db_metadata ( - singleton, - time_created, - time_modified, - version, - target_version -) VALUES - ( TRUE, NOW(), NOW(), '7.0.0', NULL) -ON CONFLICT DO NOTHING; - - - -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( id UUID PRIMARY KEY, @@ -2590,4 +2559,33 @@ FROM WHERE instance.time_deleted IS NULL AND vmm.time_deleted IS NULL; +CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( + -- There should only be one row of this table for the whole DB. + -- It's a little goofy, but filter on "singleton = true" before querying + -- or applying updates, and you'll access the singleton row. + -- + -- We also add a constraint on this table to ensure it's not possible to + -- access the version of this table with "singleton = false". + singleton BOOL NOT NULL PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + -- Semver representation of the DB version + version STRING(64) NOT NULL, + + -- (Optional) Semver representation of the DB version to which we're upgrading + target_version STRING(64), + + CHECK (singleton = true) +); + +INSERT INTO omicron.public.db_metadata ( + singleton, + time_created, + time_modified, + version, + target_version +) VALUES + ( TRUE, NOW(), NOW(), '7.0.0', NULL) +ON CONFLICT DO NOTHING; + COMMIT; From b5e62f612f6e629904d7b0db4d189685371a1e7a Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Wed, 18 Oct 2023 21:54:34 -0400 Subject: [PATCH 63/89] Implement image deletion (#4296) Also, make sure that multiple volume construction requests can only be copied for read-only Regions: the `region` table has a column for the volume that owns it, and this means there's a one to one mapping for the volume that references the region read-write. Ensure this in `volume_checkout_randomize_ids` otherwise multiple volumes could reference the same regions and this could lead to accounting / cleanup issues. Fixes #3033 --- nexus/db-queries/src/db/datastore/image.rs | 37 +++ nexus/db-queries/src/db/datastore/volume.rs | 18 +- nexus/src/app/image.rs | 29 +- nexus/src/app/sagas/image_delete.rs | 124 +++++++++ nexus/src/app/sagas/mod.rs | 4 + nexus/tests/integration_tests/images.rs | 128 ++++++++- nexus/tests/integration_tests/projects.rs | 32 +-- .../integration_tests/volume_management.rs | 248 +++++++++++++++++- 8 files changed, 584 insertions(+), 36 deletions(-) create mode 100644 nexus/src/app/sagas/image_delete.rs diff --git a/nexus/db-queries/src/db/datastore/image.rs b/nexus/db-queries/src/db/datastore/image.rs index e44da013cd..759b523010 100644 --- a/nexus/db-queries/src/db/datastore/image.rs +++ b/nexus/db-queries/src/db/datastore/image.rs @@ -19,6 +19,7 @@ use nexus_db_model::Name; use nexus_types::identity::Resource; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_common::api::external::UpdateResult; @@ -232,4 +233,40 @@ impl DataStore { })?; Ok(image) } + + pub async fn silo_image_delete( + &self, + opctx: &OpContext, + authz_image: &authz::SiloImage, + image: SiloImage, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_image).await?; + self.image_delete(opctx, image.into()).await + } + + pub async fn project_image_delete( + &self, + opctx: &OpContext, + authz_image: &authz::ProjectImage, + image: ProjectImage, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_image).await?; + self.image_delete(opctx, image.into()).await + } + + async fn image_delete( + &self, + opctx: &OpContext, + image: Image, + ) -> DeleteResult { + use db::schema::image::dsl; + diesel::update(dsl::image) + .filter(dsl::id.eq(image.id())) + .set(dsl::time_deleted.eq(Utc::now())) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } } diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index 38e3875036..f9f982213f 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -14,6 +14,7 @@ use crate::db::model::Dataset; use crate::db::model::Region; use crate::db::model::RegionSnapshot; use crate::db::model::Volume; +use anyhow::bail; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; @@ -391,6 +392,16 @@ impl DataStore { opts, gen, } => { + if !opts.read_only { + // Only one volume can "own" a Region, and that volume's + // UUID is recorded in the region table accordingly. It is + // an error to make a copy of a volume construction request + // that references non-read-only Regions. + bail!( + "only one Volume can reference a Region non-read-only!" + ); + } + let mut opts = opts.clone(); opts.id = Uuid::new_v4(); @@ -416,7 +427,10 @@ impl DataStore { /// Checkout a copy of the Volume from the database using `volume_checkout`, /// then randomize the UUIDs in the construction request. Because this is a /// new volume, it is immediately passed to `volume_create` so that the - /// accounting for Crucible resources stays correct. + /// accounting for Crucible resources stays correct. This is only valid for + /// Volumes that reference regions read-only - it's important for accounting + /// purposes that each region in this volume construction request is + /// returned by `read_only_resources_associated_with_volume`. pub async fn volume_checkout_randomize_ids( &self, volume_id: Uuid, @@ -469,6 +483,8 @@ impl DataStore { .eq(0) // Despite the SQL specifying that this column is NOT NULL, // this null check is required for this function to work! + // It's possible that the left join of region_snapshot above + // could join zero rows, making this null. .or(dsl::volume_references.is_null()), ) // where the volume has already been soft-deleted diff --git a/nexus/src/app/image.rs b/nexus/src/app/image.rs index ac51773b05..8fa9308c1d 100644 --- a/nexus/src/app/image.rs +++ b/nexus/src/app/image.rs @@ -4,8 +4,8 @@ //! Images (both project and silo scoped) -use super::Unimpl; use crate::external_api::params; +use nexus_db_queries::authn; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -27,6 +27,8 @@ use std::str::FromStr; use std::sync::Arc; use uuid::Uuid; +use super::sagas; + impl super::Nexus { pub(crate) async fn image_lookup<'a>( &'a self, @@ -356,26 +358,33 @@ impl super::Nexus { } } - // TODO-MVP: Implement pub(crate) async fn image_delete( self: &Arc, opctx: &OpContext, image_lookup: &ImageLookup<'_>, ) -> DeleteResult { - match image_lookup { + let image_param: sagas::image_delete::ImageParam = match image_lookup { ImageLookup::ProjectImage(lookup) => { - lookup.lookup_for(authz::Action::Delete).await?; + let (_, _, authz_image, image) = + lookup.fetch_for(authz::Action::Delete).await?; + sagas::image_delete::ImageParam::Project { authz_image, image } } ImageLookup::SiloImage(lookup) => { - lookup.lookup_for(authz::Action::Delete).await?; + let (_, authz_image, image) = + lookup.fetch_for(authz::Action::Delete).await?; + sagas::image_delete::ImageParam::Silo { authz_image, image } } }; - let error = Error::InternalError { - internal_message: "Endpoint not implemented".to_string(), + + let saga_params = sagas::image_delete::Params { + serialized_authn: authn::saga::Serialized::for_opctx(opctx), + image_param, }; - Err(self - .unimplemented_todo(opctx, Unimpl::ProtectedLookup(error)) - .await) + + self.execute_saga::(saga_params) + .await?; + + Ok(()) } /// Converts a project scoped image into a silo scoped image diff --git a/nexus/src/app/sagas/image_delete.rs b/nexus/src/app/sagas/image_delete.rs new file mode 100644 index 0000000000..1d88ff17af --- /dev/null +++ b/nexus/src/app/sagas/image_delete.rs @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::{ActionRegistry, NexusActionContext, NexusSaga}; +use crate::app::sagas; +use crate::app::sagas::declare_saga_actions; +use nexus_db_queries::{authn, authz, db}; +use serde::Deserialize; +use serde::Serialize; +use steno::ActionError; +use steno::Node; +use uuid::Uuid; + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) enum ImageParam { + Project { authz_image: authz::ProjectImage, image: db::model::ProjectImage }, + + Silo { authz_image: authz::SiloImage, image: db::model::SiloImage }, +} + +impl ImageParam { + fn volume_id(&self) -> Uuid { + match self { + ImageParam::Project { image, .. } => image.volume_id, + + ImageParam::Silo { image, .. } => image.volume_id, + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) struct Params { + pub serialized_authn: authn::saga::Serialized, + pub image_param: ImageParam, +} + +declare_saga_actions! { + image_delete; + DELETE_IMAGE_RECORD -> "no_result1" { + + sid_delete_image_record + } +} + +#[derive(Debug)] +pub(crate) struct SagaImageDelete; +impl NexusSaga for SagaImageDelete { + const NAME: &'static str = "image-delete"; + type Params = Params; + + fn register_actions(registry: &mut ActionRegistry) { + image_delete_register_actions(registry); + } + + fn make_saga_dag( + params: &Self::Params, + mut builder: steno::DagBuilder, + ) -> Result { + builder.append(delete_image_record_action()); + + const DELETE_VOLUME_PARAMS: &'static str = "delete_volume_params"; + + let volume_delete_params = sagas::volume_delete::Params { + serialized_authn: params.serialized_authn.clone(), + volume_id: params.image_param.volume_id(), + }; + builder.append(Node::constant( + DELETE_VOLUME_PARAMS, + serde_json::to_value(&volume_delete_params).map_err(|e| { + super::SagaInitError::SerializeError( + String::from("volume_id"), + e, + ) + })?, + )); + + let make_volume_delete_dag = || { + let subsaga_builder = steno::DagBuilder::new(steno::SagaName::new( + sagas::volume_delete::SagaVolumeDelete::NAME, + )); + sagas::volume_delete::create_dag(subsaga_builder) + }; + builder.append(steno::Node::subsaga( + "delete_volume", + make_volume_delete_dag()?, + DELETE_VOLUME_PARAMS, + )); + + Ok(builder.build()?) + } +} + +// image delete saga: action implementations + +async fn sid_delete_image_record( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + match params.image_param { + ImageParam::Project { authz_image, image } => { + osagactx + .datastore() + .project_image_delete(&opctx, &authz_image, image) + .await + .map_err(ActionError::action_failed)?; + } + + ImageParam::Silo { authz_image, image } => { + osagactx + .datastore() + .silo_image_delete(&opctx, &authz_image, image) + .await + .map_err(ActionError::action_failed)?; + } + } + + Ok(()) +} diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs index 88778e3573..0ae17c7237 100644 --- a/nexus/src/app/sagas/mod.rs +++ b/nexus/src/app/sagas/mod.rs @@ -22,6 +22,7 @@ use uuid::Uuid; pub mod disk_create; pub mod disk_delete; pub mod finalize_disk; +pub mod image_delete; pub mod import_blocks_from_url; mod instance_common; pub mod instance_create; @@ -167,6 +168,9 @@ fn make_action_registry() -> ActionRegistry { &mut registry, ); ::register_actions(&mut registry); + ::register_actions( + &mut registry, + ); #[cfg(test)] ::register_actions(&mut registry); diff --git a/nexus/tests/integration_tests/images.rs b/nexus/tests/integration_tests/images.rs index 84a8a1f50f..c3db9e8f13 100644 --- a/nexus/tests/integration_tests/images.rs +++ b/nexus/tests/integration_tests/images.rs @@ -7,15 +7,21 @@ use dropshot::ResultsPage; use http::method::Method; use http::StatusCode; +use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; +use nexus_db_queries::db::fixed_data::silo_user::USER_TEST_UNPRIVILEGED; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils::resource_helpers::grant_iam; use nexus_test_utils::resource_helpers::DiskTest; use nexus_test_utils_macros::nexus_test; -use omicron_common::api::external::Disk; - +use nexus_types::external_api::shared::ProjectRole; +use nexus_types::external_api::shared::SiloRole; use nexus_types::external_api::{params, views}; +use nexus_types::identity::Asset; +use nexus_types::identity::Resource; +use omicron_common::api::external::Disk; use omicron_common::api::external::{ByteCount, IdentityMetadataCreateParams}; use httptest::{matchers::*, responders::*, Expectation, ServerBuilder}; @@ -709,3 +715,121 @@ async fn test_image_from_other_project_snapshot_fails( .unwrap(); assert_eq!(error.message, "snapshot does not belong to this project"); } + +#[nexus_test] +async fn test_image_deletion_permissions(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + DiskTest::new(&cptestctx).await; + + // Create a project + + create_project(client, PROJECT_NAME).await; + + // Grant the unprivileged user viewer on the silo and admin on that project + + let silo_url = format!("/v1/system/silos/{}", DEFAULT_SILO.id()); + grant_iam( + client, + &silo_url, + SiloRole::Viewer, + USER_TEST_UNPRIVILEGED.id(), + AuthnMode::PrivilegedUser, + ) + .await; + + let project_url = format!("/v1/projects/{}", PROJECT_NAME); + grant_iam( + client, + &project_url, + ProjectRole::Admin, + USER_TEST_UNPRIVILEGED.id(), + AuthnMode::PrivilegedUser, + ) + .await; + + // Create an image in the default silo using the privileged user + + let server = ServerBuilder::new().run().unwrap(); + server.expect( + Expectation::matching(request::method_path("HEAD", "/image.raw")) + .times(1..) + .respond_with( + status_code(200).append_header( + "Content-Length", + format!("{}", 4096 * 1000), + ), + ), + ); + + let silo_images_url = "/v1/images"; + let images_url = get_project_images_url(PROJECT_NAME); + + let image_create_params = get_image_create(params::ImageSource::Url { + url: server.url("/image.raw").to_string(), + block_size: BLOCK_SIZE, + }); + + let image = + NexusRequest::objects_post(client, &images_url, &image_create_params) + .authn_as(AuthnMode::PrivilegedUser) + .execute_and_parse_unwrap::() + .await; + + let image_id = image.identity.id; + + // promote the image to the silo + + let promote_url = format!("/v1/images/{}/promote", image_id); + NexusRequest::new( + RequestBuilder::new(client, http::Method::POST, &promote_url) + .expect_status(Some(http::StatusCode::ACCEPTED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute_and_parse_unwrap::() + .await; + + let silo_images = NexusRequest::object_get(client, &silo_images_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute_and_parse_unwrap::>() + .await + .items; + + assert_eq!(silo_images.len(), 1); + assert_eq!(silo_images[0].identity.name, "alpine-edge"); + + // the unprivileged user should not be able to delete that image + + let image_url = format!("/v1/images/{}", image_id); + NexusRequest::new( + RequestBuilder::new(client, http::Method::DELETE, &image_url) + .expect_status(Some(http::StatusCode::FORBIDDEN)), + ) + .authn_as(AuthnMode::UnprivilegedUser) + .execute() + .await + .expect("should not be able to delete silo image as unpriv user!"); + + // Demote that image + + let demote_url = + format!("/v1/images/{}/demote?project={}", image_id, PROJECT_NAME); + NexusRequest::new( + RequestBuilder::new(client, http::Method::POST, &demote_url) + .expect_status(Some(http::StatusCode::ACCEPTED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute_and_parse_unwrap::() + .await; + + // now the unpriviledged user should be able to delete that image + + let image_url = format!("/v1/images/{}", image_id); + NexusRequest::new( + RequestBuilder::new(client, http::Method::DELETE, &image_url) + .expect_status(Some(http::StatusCode::NO_CONTENT)), + ) + .authn_as(AuthnMode::UnprivilegedUser) + .execute() + .await + .expect("should be able to delete project image as unpriv user!"); +} diff --git a/nexus/tests/integration_tests/projects.rs b/nexus/tests/integration_tests/projects.rs index f974e85dc4..24b2721a1d 100644 --- a/nexus/tests/integration_tests/projects.rs +++ b/nexus/tests/integration_tests/projects.rs @@ -245,32 +245,24 @@ async fn test_project_deletion_with_image(cptestctx: &ControlPlaneTestContext) { delete_project_expect_fail(&url, &client).await, ); - // TODO: finish test once image delete is implemented. Image create works - // and project delete with image fails as expected, but image delete is not - // implemented yet, so we can't show that project delete works after image - // delete. let image_url = format!("/v1/images/{}", image.identity.id); - NexusRequest::expect_failure_with_body( - client, - StatusCode::INTERNAL_SERVER_ERROR, - Method::DELETE, - &image_url, - &image_create_params, + NexusRequest::object_delete(&client, &image_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete image"); + + // Expect that trying to GET the image results in a 404 + NexusRequest::new( + RequestBuilder::new(&client, http::Method::GET, &image_url) + .expect_status(Some(http::StatusCode::NOT_FOUND)), ) .authn_as(AuthnMode::PrivilegedUser) .execute() .await - .unwrap(); + .expect("GET of a deleted image did not return 404"); - // TODO: delete the image - // NexusRequest::object_delete(&client, &image_url) - // .authn_as(AuthnMode::PrivilegedUser) - // .execute() - // .await - // .expect("failed to delete image"); - - // TODO: now delete project works - // delete_project(&url, &client).await; + delete_project(&url, &client).await; } #[nexus_test] diff --git a/nexus/tests/integration_tests/volume_management.rs b/nexus/tests/integration_tests/volume_management.rs index e263593def..f5935676a7 100644 --- a/nexus/tests/integration_tests/volume_management.rs +++ b/nexus/tests/integration_tests/volume_management.rs @@ -1103,8 +1103,6 @@ async fn test_create_image_from_snapshot_delete( assert!(!disk_test.crucible_resources_deleted().await); // Delete the image - // TODO-unimplemented - /* let image_url = "/v1/images/debian-11"; NexusRequest::object_delete(client, &image_url) .authn_as(AuthnMode::PrivilegedUser) @@ -1114,7 +1112,213 @@ async fn test_create_image_from_snapshot_delete( // Assert everything was cleaned up assert!(disk_test.crucible_resources_deleted().await); - */ +} + +enum DeleteImageTestParam { + Image, + Disk, + Snapshot, +} + +async fn delete_image_test( + cptestctx: &ControlPlaneTestContext, + order: &[DeleteImageTestParam], +) { + // 1. Create a blank disk + // 2. Take a snapshot of that disk + // 3. Create an image from that snapshot + // 4. Delete each of these items in some order + + let disk_test = DiskTest::new(&cptestctx).await; + + let client = &cptestctx.external_client; + populate_ip_pool(&client, "default", None).await; + create_org_and_project(client).await; + + let disks_url = get_disks_url(); + + // Create a blank disk + + let disk_size = ByteCount::from_gibibytes_u32(2); + let base_disk_name: Name = "base-disk".parse().unwrap(); + let base_disk = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: base_disk_name.clone(), + description: String::from("all your base disk are belong to us"), + }, + disk_source: params::DiskSource::Blank { + block_size: params::BlockSize::try_from(512).unwrap(), + }, + size: disk_size, + }; + + let _base_disk: Disk = NexusRequest::new( + RequestBuilder::new(client, Method::POST, &disks_url) + .body(Some(&base_disk)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + + // Issue snapshot request + let snapshots_url = format!("/v1/snapshots?project={}", PROJECT_NAME); + + let snapshot: views::Snapshot = object_create( + client, + &snapshots_url, + ¶ms::SnapshotCreate { + identity: IdentityMetadataCreateParams { + name: "a-snapshot".parse().unwrap(), + description: String::from("you are on the way to destruction"), + }, + disk: base_disk_name.clone().into(), + }, + ) + .await; + + // Create an image from the snapshot + let image_create_params = params::ImageCreate { + identity: IdentityMetadataCreateParams { + name: "debian-11".parse().unwrap(), + description: String::from( + "you have no chance to survive make your time", + ), + }, + source: params::ImageSource::Snapshot { id: snapshot.identity.id }, + os: "debian".parse().unwrap(), + version: "12".into(), + }; + + let _image: views::Image = + NexusRequest::objects_post(client, "/v1/images", &image_create_params) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + + assert_eq!(order.len(), 3); + for item in order { + // Still some crucible resources + assert!(!disk_test.crucible_resources_deleted().await); + + match item { + DeleteImageTestParam::Image => { + let image_url = "/v1/images/debian-11"; + NexusRequest::object_delete(client, &image_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete image"); + } + + DeleteImageTestParam::Disk => { + NexusRequest::object_delete(client, &get_disk_url("base-disk")) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete disk"); + } + + DeleteImageTestParam::Snapshot => { + let snapshot_url = get_snapshot_url("a-snapshot"); + NexusRequest::object_delete(client, &snapshot_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete snapshot"); + } + } + } + + // Assert everything was cleaned up + assert!(disk_test.crucible_resources_deleted().await); +} + +// Make sure that whatever order disks, images, and snapshots are deleted, the +// Crucible resource accounting that Nexus does is correct. + +#[nexus_test] +async fn test_delete_image_order_1(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Disk, + DeleteImageTestParam::Image, + DeleteImageTestParam::Snapshot, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_2(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Disk, + DeleteImageTestParam::Snapshot, + DeleteImageTestParam::Image, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_3(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Image, + DeleteImageTestParam::Disk, + DeleteImageTestParam::Snapshot, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_4(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Image, + DeleteImageTestParam::Snapshot, + DeleteImageTestParam::Disk, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_5(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Snapshot, + DeleteImageTestParam::Disk, + DeleteImageTestParam::Image, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_6(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Snapshot, + DeleteImageTestParam::Image, + DeleteImageTestParam::Disk, + ], + ) + .await; } // A test function to create a volume with the provided read only parent. @@ -1814,6 +2018,44 @@ async fn test_volume_checkout_updates_sparse_mid_multiple_gen( volume_match_gen(new_vol, vec![Some(8), None, Some(10)]); } +#[nexus_test] +async fn test_volume_checkout_randomize_ids_only_read_only( + cptestctx: &ControlPlaneTestContext, +) { + // Verify that a volume_checkout_randomize_ids will not work for + // non-read-only Regions + let nexus = &cptestctx.server.apictx().nexus; + let datastore = nexus.datastore(); + let volume_id = Uuid::new_v4(); + let block_size = 512; + + // Create three sub_vols. + let subvol_one = create_region(block_size, 7, Uuid::new_v4()); + let subvol_two = create_region(block_size, 7, Uuid::new_v4()); + let subvol_three = create_region(block_size, 7, Uuid::new_v4()); + + // Make the volume with our three sub_volumes + let volume_construction_request = VolumeConstructionRequest::Volume { + id: volume_id, + block_size, + sub_volumes: vec![subvol_one, subvol_two, subvol_three], + read_only_parent: None, + }; + + // Insert the volume into the database. + datastore + .volume_create(nexus_db_model::Volume::new( + volume_id, + serde_json::to_string(&volume_construction_request).unwrap(), + )) + .await + .unwrap(); + + // volume_checkout_randomize_ids should fail + let r = datastore.volume_checkout_randomize_ids(volume_id).await; + assert!(r.is_err()); +} + /// Test that the Crucible agent's port reuse does not confuse /// `decrease_crucible_resource_count_and_soft_delete_volume`, due to the /// `[ipv6]:port` targets being reused. From 93b280ce4d259ea2449cdf498f7e7b843cb46cf8 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Thu, 19 Oct 2023 08:36:39 -0700 Subject: [PATCH 64/89] plumb customer-configured DNS into instances (#4265) Currently [OPTE hardcodes DNS servers to 8.8.8.8](https://github.com/oxidecomputer/opte/issues/390), and we would prefer to not do this. When setting up the rack, customers configure DNS servers. Currently this is added to Nexus's deployment configuration, so that Nexus can resolve external domain names for SAML. This change plumbs these DNS servers through to sled-agent and OPTE. For non-instance OPTE ports, `DhcpCfg::default()` is used, which is equivalent to "no hostname, no domain name, no DNS servers, no search domains". This is not the final state of the DHCP work we want to do; in talking with @rmustacc and @rcgoodfellow we agree that implementing this minimal DHCP option set is the correct thing to do urgently; and that we still need to design the broader inter-instance networking picture (whether we want an instance-facing recursive resolver on the rack and whether it should resolve domain names for other instances with VPC IPs, what those hostnames and search domains should be, and how customers can modify these settings). --- .github/buildomat/jobs/deploy.sh | 2 +- Cargo.lock | 13 ++++----- Cargo.toml | 4 +-- illumos-utils/src/opte/mod.rs | 1 + illumos-utils/src/opte/params.rs | 23 +++++++++++++++ illumos-utils/src/opte/port_manager.rs | 12 ++++++-- nexus/src/app/instance.rs | 6 ++++ nexus/src/app/mod.rs | 12 +++++++- openapi/sled-agent.json | 34 ++++++++++++++++++++++ sled-agent/Cargo.toml | 1 - sled-agent/src/instance.rs | 40 +++++++++++++++++++++++++- sled-agent/src/params.rs | 2 ++ sled-agent/src/services.rs | 10 +++---- tools/opte_version | 2 +- 14 files changed, 140 insertions(+), 22 deletions(-) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index c2579d98ea..bdc1a9cce8 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -2,7 +2,7 @@ #: #: name = "helios / deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.23" +#: target = "lab-2.0-opte-0.25" #: output_rules = [ #: "%/var/svc/log/oxide-sled-agent:default.log*", #: "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*", diff --git a/Cargo.lock b/Cargo.lock index 835fab53cb..43ccc57a73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3406,7 +3406,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" [[package]] name = "illumos-utils" @@ -3828,7 +3828,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "quote", "syn 1.0.109", @@ -5311,7 +5311,6 @@ dependencies = [ "openapi-lint", "openapiv3", "opte-ioctl", - "oxide-vpc", "oximeter 0.1.0", "oximeter-producer 0.1.0", "percent-encoding", @@ -5600,7 +5599,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "cfg-if 0.1.10", "dyn-clone", @@ -5617,7 +5616,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "cfg-if 0.1.10", "illumos-sys-hdrs", @@ -5630,7 +5629,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "libc", "libnet", @@ -5710,7 +5709,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "cfg-if 0.1.10", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 72a7f6157e..7e6a2b3902 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -249,7 +249,7 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.8.3" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "631c2017f19cafb1535f621e9e5aa9198ccad869", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649", features = [ "api", "std" ] } once_cell = "1.18.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "1.0" @@ -257,7 +257,7 @@ openapiv3 = "1.0" openssl = "0.10" openssl-sys = "0.9" openssl-probe = "0.1.2" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "631c2017f19cafb1535f621e9e5aa9198ccad869" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" } oso = "0.26" owo-colors = "3.5.0" oximeter = { path = "oximeter/oximeter" } diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 10e2a45d83..710e783181 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -25,6 +25,7 @@ pub use port_manager::PortTicket; use ipnetwork::IpNetwork; use macaddr::MacAddr6; pub use oxide_vpc::api::BoundaryServices; +pub use oxide_vpc::api::DhcpCfg; pub use oxide_vpc::api::Vni; use std::net::IpAddr; diff --git a/illumos-utils/src/opte/params.rs b/illumos-utils/src/opte/params.rs index 4df437546c..df1f33cb92 100644 --- a/illumos-utils/src/opte/params.rs +++ b/illumos-utils/src/opte/params.rs @@ -50,3 +50,26 @@ pub struct DeleteVirtualNetworkInterfaceHost { /// be deleted. pub vni: external::Vni, } + +/// DHCP configuration for a port +/// +/// Not present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we +/// use `InstanceRuntimeState::hostname` for this value. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct DhcpConfig { + /// DNS servers to send to the instance + /// + /// (DHCPv4 option 6; DHCPv6 option 23) + pub dns_servers: Vec, + + /// DNS zone this instance's hostname belongs to (e.g. the `project.example` + /// part of `instance1.project.example`) + /// + /// (DHCPv4 option 15; used in DHCPv6 option 39) + pub host_domain: Option, + + /// DNS search domains + /// + /// (DHCPv4 option 119; DHCPv6 option 24) + pub search_domains: Vec, +} diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 893db9a6ed..f0a8d8d839 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -19,6 +19,7 @@ use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_common::api::internal::shared::SourceNatConfig; use oxide_vpc::api::AddRouterEntryReq; +use oxide_vpc::api::DhcpCfg; use oxide_vpc::api::IpCfg; use oxide_vpc::api::IpCidr; use oxide_vpc::api::Ipv4Cfg; @@ -100,6 +101,7 @@ impl PortManager { source_nat: Option, external_ips: &[IpAddr], firewall_rules: &[VpcFirewallRule], + dhcp_config: DhcpCfg, ) -> Result<(Port, PortTicket), Error> { let mac = *nic.mac; let vni = Vni::new(nic.vni).unwrap(); @@ -205,8 +207,6 @@ impl PortManager { vni, phys_ip: self.inner.underlay_ip.into(), boundary_services, - // TODO-completeness (#2153): Plumb domain search list - domain_list: vec![], }; // Create the xde device. @@ -227,11 +227,17 @@ impl PortManager { "Creating xde device"; "port_name" => &port_name, "vpc_cfg" => ?&vpc_cfg, + "dhcp_config" => ?&dhcp_config, ); #[cfg(target_os = "illumos")] let hdl = { let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?; - hdl.create_xde(&port_name, vpc_cfg, /* passthru = */ false)?; + hdl.create_xde( + &port_name, + vpc_cfg, + dhcp_config, + /* passthru = */ false, + )?; hdl }; diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 592e1f0492..1adcd8f9c0 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -987,6 +987,12 @@ impl super::Nexus { source_nat, external_ips, firewall_rules, + dhcp_config: sled_agent_client::types::DhcpConfig { + dns_servers: self.external_dns_servers.clone(), + // TODO: finish designing instance DNS + host_domain: None, + search_domains: Vec::new(), + }, disks: disk_reqs, cloud_init_bytes: Some(base64::Engine::encode( &base64::engine::general_purpose::STANDARD, diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 45f69848e3..23ded83150 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -26,7 +26,7 @@ use omicron_common::api::internal::shared::SwitchLocation; use omicron_common::nexus_config::RegionAllocationStrategy; use slog::Logger; use std::collections::HashMap; -use std::net::Ipv6Addr; +use std::net::{IpAddr, Ipv6Addr}; use std::sync::Arc; use uuid::Uuid; @@ -153,6 +153,12 @@ pub struct Nexus { /// DNS resolver Nexus uses to resolve an external host external_resolver: Arc, + /// DNS servers used in `external_resolver`, used to provide DNS servers to + /// instances via DHCP + // TODO: This needs to be moved to the database. + // https://github.com/oxidecomputer/omicron/issues/3732 + external_dns_servers: Vec, + /// Mapping of SwitchLocations to their respective Dendrite Clients dpd_clients: HashMap>, @@ -332,6 +338,10 @@ impl Nexus { samael_max_issue_delay: std::sync::Mutex::new(None), internal_resolver: resolver, external_resolver, + external_dns_servers: config + .deployment + .external_dns_servers + .clone(), dpd_clients, background_tasks, default_region_allocation_strategy: config diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 56437ab283..7831193fc2 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -1194,6 +1194,36 @@ "vni" ] }, + "DhcpConfig": { + "description": "DHCP configuration for a port\n\nNot present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we use `InstanceRuntimeState::hostname` for this value.", + "type": "object", + "properties": { + "dns_servers": { + "description": "DNS servers to send to the instance\n\n(DHCPv4 option 6; DHCPv6 option 23)", + "type": "array", + "items": { + "type": "string", + "format": "ip" + } + }, + "host_domain": { + "nullable": true, + "description": "DNS zone this instance's hostname belongs to (e.g. the `project.example` part of `instance1.project.example`)\n\n(DHCPv4 option 15; used in DHCPv6 option 39)", + "type": "string" + }, + "search_domains": { + "description": "DNS search domains\n\n(DHCPv4 option 119; DHCPv6 option 24)", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "dns_servers", + "search_domains" + ] + }, "DiskEnsureBody": { "description": "Sent from to a sled agent to establish the runtime state of a Disk", "type": "object", @@ -1697,6 +1727,9 @@ "nullable": true, "type": "string" }, + "dhcp_config": { + "$ref": "#/components/schemas/DhcpConfig" + }, "disks": { "type": "array", "items": { @@ -1731,6 +1764,7 @@ } }, "required": [ + "dhcp_config", "disks", "external_ips", "firewall_rules", diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 82d7411d1a..636c9665ef 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -44,7 +44,6 @@ macaddr.workspace = true nexus-client.workspace = true omicron-common.workspace = true once_cell.workspace = true -oxide-vpc.workspace = true oximeter.workspace = true oximeter-producer.workspace = true percent-encoding.workspace = true diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index ce1ef662dc..94614c2363 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -26,7 +26,7 @@ use chrono::Utc; use futures::lock::{Mutex, MutexGuard}; use illumos_utils::dladm::Etherstub; use illumos_utils::link::VnicAllocator; -use illumos_utils::opte::PortManager; +use illumos_utils::opte::{DhcpCfg, PortManager}; use illumos_utils::running_zone::{InstalledZone, RunningZone}; use illumos_utils::svc::wait_for_service; use illumos_utils::zone::Zones; @@ -91,6 +91,10 @@ pub enum Error { #[error(transparent)] Opte(#[from] illumos_utils::opte::Error), + /// Issued by `impl TryFrom<&[u8]> for oxide_vpc::api::DomainName` + #[error("Invalid hostname: {0}")] + InvalidHostname(&'static str), + #[error("Error resolving DNS name: {0}")] ResolveError(#[from] internal_dns::resolver::ResolveError), @@ -207,6 +211,7 @@ struct InstanceInner { source_nat: SourceNatConfig, external_ips: Vec, firewall_rules: Vec, + dhcp_config: DhcpCfg, // Disk related properties // TODO: replace `propolis_client::handmade::*` with properly-modeled local types @@ -610,6 +615,37 @@ impl Instance { zone_bundler, } = services; + let mut dhcp_config = DhcpCfg { + hostname: Some( + hardware + .properties + .hostname + .parse() + .map_err(Error::InvalidHostname)?, + ), + host_domain: hardware + .dhcp_config + .host_domain + .map(|domain| domain.parse()) + .transpose() + .map_err(Error::InvalidHostname)?, + domain_search_list: hardware + .dhcp_config + .search_domains + .into_iter() + .map(|domain| domain.parse()) + .collect::>() + .map_err(Error::InvalidHostname)?, + dns4_servers: Vec::new(), + dns6_servers: Vec::new(), + }; + for ip in hardware.dhcp_config.dns_servers { + match ip { + IpAddr::V4(ip) => dhcp_config.dns4_servers.push(ip.into()), + IpAddr::V6(ip) => dhcp_config.dns6_servers.push(ip.into()), + } + } + let instance = InstanceInner { log: log.new(o!("instance_id" => id.to_string())), // NOTE: Mostly lies. @@ -633,6 +669,7 @@ impl Instance { source_nat: hardware.source_nat, external_ips: hardware.external_ips, firewall_rules: hardware.firewall_rules, + dhcp_config, requested_disks: hardware.disks, cloud_init_bytes: hardware.cloud_init_bytes, state: InstanceStates::new( @@ -852,6 +889,7 @@ impl Instance { snat, external_ips, &inner.firewall_rules, + inner.dhcp_config.clone(), )?; opte_ports.push(port); } diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 84ec1ef0dc..e1c8b05cde 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -6,6 +6,7 @@ use crate::zone_bundle::PriorityOrder; pub use crate::zone_bundle::ZoneBundleCause; pub use crate::zone_bundle::ZoneBundleId; pub use crate::zone_bundle::ZoneBundleMetadata; +pub use illumos_utils::opte::params::DhcpConfig; pub use illumos_utils::opte::params::VpcFirewallRule; pub use illumos_utils::opte::params::VpcFirewallRulesEnsureBody; use omicron_common::api::internal::nexus::{ @@ -68,6 +69,7 @@ pub struct InstanceHardware { /// provided to an instance to allow inbound connectivity. pub external_ips: Vec, pub firewall_rules: Vec, + pub dhcp_config: DhcpConfig, // TODO: replace `propolis_client::handmade::*` with locally-modeled request type pub disks: Vec, pub cloud_init_bytes: Option, diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index f91b5091e6..06d3ae1977 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -52,7 +52,7 @@ use illumos_utils::dladm::{ Dladm, Etherstub, EtherstubVnic, GetSimnetError, PhysicalLink, }; use illumos_utils::link::{Link, VnicAllocator}; -use illumos_utils::opte::{Port, PortManager, PortTicket}; +use illumos_utils::opte::{DhcpCfg, Port, PortManager, PortTicket}; use illumos_utils::running_zone::{ InstalledZone, RunCommandError, RunningZone, }; @@ -863,11 +863,11 @@ impl ServiceManager { // config allows outbound access which is enough for // Boundary NTP which needs to come up before Nexus. let port = port_manager - .create_port(nic, snat, external_ips, &[]) + .create_port(nic, snat, external_ips, &[], DhcpCfg::default()) .map_err(|err| Error::ServicePortCreation { - service: svc.details.to_string(), - err: Box::new(err), - })?; + service: svc.details.to_string(), + err: Box::new(err), + })?; // We also need to update the switch with the NAT mappings let (target_ip, first_port, last_port) = match snat { diff --git a/tools/opte_version b/tools/opte_version index 2dbaeb7154..0a79a6aba9 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.23.181 +0.25.183 From 58c8c6ea74ae459743b27e67c6f8d2de1ce9bb7a Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Thu, 19 Oct 2023 10:24:16 -0700 Subject: [PATCH 65/89] [wicketd] Accept TUF repos with RoT archives signed with different keys (#4289) As of this PR, wicketd will (a) accept TUF repos containing multiple RoT archives for the same board target (e.g., multiple gimlet RoT images), and when performing a mupdate, it will ask the RoT for its currently-active CMPA and CFPA pages and search for an RoT archive that matches. After this is deployed to all fielded systems, we'll be able to drop the `-rot-staging-dev` and `-prod-rel` TUF repos from CI, and only build a single TUF repo with all RoT images. This PR adds a new `-rot-all` TUF repo publishing step but does not remove the old ones, as we'll need them to update into this version of wicketd. --- .github/buildomat/jobs/tuf-repo.sh | 62 ++++++ Cargo.lock | 52 ++--- Cargo.toml | 4 +- gateway/Cargo.toml | 1 + gateway/src/http_entrypoints.rs | 137 +++++++++++++ openapi/gateway.json | 201 +++++++++++++++++++ sp-sim/src/gimlet.rs | 19 ++ sp-sim/src/sidecar.rs | 19 ++ wicket-common/src/update_events.rs | 15 ++ wicketd/Cargo.toml | 1 + wicketd/src/artifacts/extracted_artifacts.rs | 2 +- wicketd/src/artifacts/update_plan.rs | 132 ++++++------ wicketd/src/update_tracker.rs | 171 +++++++++++++--- wicketd/tests/integration_tests/updates.rs | 4 +- workspace-hack/Cargo.toml | 4 +- 15 files changed, 695 insertions(+), 129 deletions(-) diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index 57d3ba8a1f..ea25ab5834 100644 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -25,6 +25,21 @@ #: job = "helios / build trampoline OS image" #: #: [[publish]] +#: series = "rot-all" +#: name = "repo.zip.parta" +#: from_output = "/work/repo-rot-all.zip.parta" +#: +#: [[publish]] +#: series = "rot-all" +#: name = "repo.zip.partb" +#: from_output = "/work/repo-rot-all.zip.partb" +#: +#: [[publish]] +#: series = "rot-all" +#: name = "repo.zip.sha256.txt" +#: from_output = "/work/repo-rot-all.zip.sha256.txt" +#: +#: [[publish]] #: series = "rot-prod-rel" #: name = "repo.zip.parta" #: from_output = "/work/repo-rot-prod-rel.zip.parta" @@ -168,6 +183,38 @@ caboose_util_rot() { } SERIES_LIST=() + +# Create an initial `manifest-rot-all.toml` containing the SP images for all +# boards. While we still need to build multiple TUF repos, +# `add_hubris_artifacts` below will append RoT images to this manifest (in +# addition to the single-RoT manifest it creates). +prep_rot_all_series() { + series="rot-all" + + SERIES_LIST+=("$series") + + manifest=/work/manifest-$series.toml + cp /work/manifest.toml "$manifest" + + for board_rev in "${ALL_BOARDS[@]}"; do + board=${board_rev%-?} + tufaceous_board=${board//sidecar/switch} + sp_image="/work/hubris/${board_rev}.zip" + sp_caboose_version=$(/work/caboose-util read-version "$sp_image") + sp_caboose_board=$(/work/caboose-util read-board "$sp_image") + + cat >>"$manifest" <>"$manifest_rot_all" <>, + path: Path, +) -> Result, HttpError> { + let apictx = rqctx.context(); + + let PathSpComponent { sp, component } = path.into_inner(); + + // Ensure the caller knows they're asking for the RoT + if component_from_str(&component)? != SpComponent::ROT { + return Err(HttpError::for_bad_request( + Some("RequestUnsupportedForComponent".to_string()), + "Only the RoT has a CFPA".into(), + )); + } + + let sp = apictx.mgmt_switch.sp(sp.into())?; + let data = sp.read_rot_cmpa().await.map_err(SpCommsError::from)?; + + let base64_data = base64::engine::general_purpose::STANDARD.encode(data); + + Ok(HttpResponseOk(RotCmpa { base64_data })) +} + +/// Read the requested CFPA slot from a root of trust. +/// +/// This endpoint is only valid for the `rot` component. +#[endpoint { + method = GET, + path = "/sp/{type}/{slot}/component/{component}/cfpa", +}] +async fn sp_rot_cfpa_get( + rqctx: RequestContext>, + path: Path, + params: TypedBody, +) -> Result, HttpError> { + let apictx = rqctx.context(); + + let PathSpComponent { sp, component } = path.into_inner(); + let GetCfpaParams { slot } = params.into_inner(); + + // Ensure the caller knows they're asking for the RoT + if component_from_str(&component)? != SpComponent::ROT { + return Err(HttpError::for_bad_request( + Some("RequestUnsupportedForComponent".to_string()), + "Only the RoT has a CFPA".into(), + )); + } + + let sp = apictx.mgmt_switch.sp(sp.into())?; + let data = match slot { + RotCfpaSlot::Active => sp.read_rot_active_cfpa().await, + RotCfpaSlot::Inactive => sp.read_rot_inactive_cfpa().await, + RotCfpaSlot::Scratch => sp.read_rot_scratch_cfpa().await, + } + .map_err(SpCommsError::from)?; + + let base64_data = base64::engine::general_purpose::STANDARD.encode(data); + + Ok(HttpResponseOk(RotCfpa { base64_data, slot })) +} + /// List SPs via Ignition /// /// Retreive information for all SPs via the Ignition controller. This is lower @@ -1324,6 +1459,8 @@ pub fn api() -> GatewayApiDescription { api.register(sp_component_update)?; api.register(sp_component_update_status)?; api.register(sp_component_update_abort)?; + api.register(sp_rot_cmpa_get)?; + api.register(sp_rot_cfpa_get)?; api.register(sp_host_phase2_progress_get)?; api.register(sp_host_phase2_progress_delete)?; api.register(ignition_list)?; diff --git a/openapi/gateway.json b/openapi/gateway.json index 847d1f746d..67cc2bd634 100644 --- a/openapi/gateway.json +++ b/openapi/gateway.json @@ -551,6 +551,70 @@ } } }, + "/sp/{type}/{slot}/component/{component}/cfpa": { + "get": { + "summary": "Read the requested CFPA slot from a root of trust.", + "description": "This endpoint is only valid for the `rot` component.", + "operationId": "sp_rot_cfpa_get", + "parameters": [ + { + "in": "path", + "name": "component", + "description": "ID for the component of the SP; this is the internal identifier used by the SP itself to identify its components.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "slot", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "in": "path", + "name": "type", + "required": true, + "schema": { + "$ref": "#/components/schemas/SpType" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetCfpaParams" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RotCfpa" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/sp/{type}/{slot}/component/{component}/clear-status": { "post": { "summary": "Clear status of a component", @@ -598,6 +662,60 @@ } } }, + "/sp/{type}/{slot}/component/{component}/cmpa": { + "get": { + "summary": "Read the CMPA from a root of trust.", + "description": "This endpoint is only valid for the `rot` component.", + "operationId": "sp_rot_cmpa_get", + "parameters": [ + { + "in": "path", + "name": "component", + "description": "ID for the component of the SP; this is the internal identifier used by the SP itself to identify its components.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "slot", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "in": "path", + "name": "type", + "required": true, + "schema": { + "$ref": "#/components/schemas/SpType" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RotCmpa" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/sp/{type}/{slot}/component/{component}/reset": { "post": { "summary": "Reset an SP component (possibly the SP itself).", @@ -1326,6 +1444,17 @@ "request_id" ] }, + "GetCfpaParams": { + "type": "object", + "properties": { + "slot": { + "$ref": "#/components/schemas/RotCfpaSlot" + } + }, + "required": [ + "slot" + ] + }, "HostPhase2Progress": { "oneOf": [ { @@ -2071,6 +2200,78 @@ "A2" ] }, + "RotCfpa": { + "type": "object", + "properties": { + "base64_data": { + "type": "string" + }, + "slot": { + "$ref": "#/components/schemas/RotCfpaSlot" + } + }, + "required": [ + "base64_data", + "slot" + ] + }, + "RotCfpaSlot": { + "oneOf": [ + { + "type": "object", + "properties": { + "slot": { + "type": "string", + "enum": [ + "active" + ] + } + }, + "required": [ + "slot" + ] + }, + { + "type": "object", + "properties": { + "slot": { + "type": "string", + "enum": [ + "inactive" + ] + } + }, + "required": [ + "slot" + ] + }, + { + "type": "object", + "properties": { + "slot": { + "type": "string", + "enum": [ + "scratch" + ] + } + }, + "required": [ + "slot" + ] + } + ] + }, + "RotCmpa": { + "type": "object", + "properties": { + "base64_data": { + "type": "string" + } + }, + "required": [ + "base64_data" + ] + }, "RotSlot": { "oneOf": [ { diff --git a/sp-sim/src/gimlet.rs b/sp-sim/src/gimlet.rs index dad53f3848..d131696559 100644 --- a/sp-sim/src/gimlet.rs +++ b/sp-sim/src/gimlet.rs @@ -1282,6 +1282,25 @@ impl SpHandler for Handler { buf[..val.len()].copy_from_slice(val); Ok(val.len()) } + + fn read_sensor( + &mut self, + _request: gateway_messages::SensorRequest, + ) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } + + fn current_time(&mut self) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } + + fn read_rot( + &mut self, + _request: gateway_messages::RotRequest, + _buf: &mut [u8], + ) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } } enum UpdateState { diff --git a/sp-sim/src/sidecar.rs b/sp-sim/src/sidecar.rs index def2a79c0c..e56c610c9c 100644 --- a/sp-sim/src/sidecar.rs +++ b/sp-sim/src/sidecar.rs @@ -1001,6 +1001,25 @@ impl SpHandler for Handler { buf[..val.len()].copy_from_slice(val); Ok(val.len()) } + + fn read_sensor( + &mut self, + _request: gateway_messages::SensorRequest, + ) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } + + fn current_time(&mut self) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } + + fn read_rot( + &mut self, + _request: gateway_messages::RotRequest, + _buf: &mut [u8], + ) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } } struct FakeIgnition { diff --git a/wicket-common/src/update_events.rs b/wicket-common/src/update_events.rs index 3dd984d07f..ac840f83ad 100644 --- a/wicket-common/src/update_events.rs +++ b/wicket-common/src/update_events.rs @@ -159,6 +159,21 @@ pub enum UpdateTerminalError { #[source] error: gateway_client::Error, }, + #[error("getting RoT CMPA failed")] + GetRotCmpaFailed { + #[source] + error: anyhow::Error, + }, + #[error("getting RoT CFPA failed")] + GetRotCfpaFailed { + #[source] + error: anyhow::Error, + }, + #[error("failed to find correctly-singed RoT image")] + FailedFindingSignedRotImage { + #[source] + error: anyhow::Error, + }, #[error("getting SP caboose failed")] GetSpCabooseFailed { #[source] diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index cf04b7c6a7..f11fda9750 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -7,6 +7,7 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true async-trait.workspace = true +base64.workspace = true bytes.workspace = true camino.workspace = true camino-tempfile.workspace = true diff --git a/wicketd/src/artifacts/extracted_artifacts.rs b/wicketd/src/artifacts/extracted_artifacts.rs index f9ad59404b..352d8ad3d5 100644 --- a/wicketd/src/artifacts/extracted_artifacts.rs +++ b/wicketd/src/artifacts/extracted_artifacts.rs @@ -61,7 +61,7 @@ impl Eq for ExtractedArtifactDataHandle {} impl ExtractedArtifactDataHandle { /// File size of this artifact in bytes. - pub(super) fn file_size(&self) -> usize { + pub(crate) fn file_size(&self) -> usize { self.file_size } diff --git a/wicketd/src/artifacts/update_plan.rs b/wicketd/src/artifacts/update_plan.rs index 2668aaac51..31a8a06ca2 100644 --- a/wicketd/src/artifacts/update_plan.rs +++ b/wicketd/src/artifacts/update_plan.rs @@ -39,14 +39,14 @@ use tufaceous_lib::RotArchives; pub struct UpdatePlan { pub(crate) system_version: SemverVersion, pub(crate) gimlet_sp: BTreeMap, - pub(crate) gimlet_rot_a: ArtifactIdData, - pub(crate) gimlet_rot_b: ArtifactIdData, + pub(crate) gimlet_rot_a: Vec, + pub(crate) gimlet_rot_b: Vec, pub(crate) psc_sp: BTreeMap, - pub(crate) psc_rot_a: ArtifactIdData, - pub(crate) psc_rot_b: ArtifactIdData, + pub(crate) psc_rot_a: Vec, + pub(crate) psc_rot_b: Vec, pub(crate) sidecar_sp: BTreeMap, - pub(crate) sidecar_rot_a: ArtifactIdData, - pub(crate) sidecar_rot_b: ArtifactIdData, + pub(crate) sidecar_rot_a: Vec, + pub(crate) sidecar_rot_b: Vec, // Note: The Trampoline image is broken into phase1/phase2 as part of our // update plan (because they go to different destinations), but the two @@ -83,14 +83,14 @@ pub(super) struct UpdatePlanBuilder<'a> { // fields that mirror `UpdatePlan` system_version: SemverVersion, gimlet_sp: BTreeMap, - gimlet_rot_a: Option, - gimlet_rot_b: Option, + gimlet_rot_a: Vec, + gimlet_rot_b: Vec, psc_sp: BTreeMap, - psc_rot_a: Option, - psc_rot_b: Option, + psc_rot_a: Vec, + psc_rot_b: Vec, sidecar_sp: BTreeMap, - sidecar_rot_a: Option, - sidecar_rot_b: Option, + sidecar_rot_a: Vec, + sidecar_rot_b: Vec, // We always send phase 1 images (regardless of host or trampoline) to the // SP via MGS, so we retain their data. @@ -124,14 +124,14 @@ impl<'a> UpdatePlanBuilder<'a> { Ok(Self { system_version, gimlet_sp: BTreeMap::new(), - gimlet_rot_a: None, - gimlet_rot_b: None, + gimlet_rot_a: Vec::new(), + gimlet_rot_b: Vec::new(), psc_sp: BTreeMap::new(), - psc_rot_a: None, - psc_rot_b: None, + psc_rot_a: Vec::new(), + psc_rot_b: Vec::new(), sidecar_sp: BTreeMap::new(), - sidecar_rot_a: None, - sidecar_rot_b: None, + sidecar_rot_a: Vec::new(), + sidecar_rot_b: Vec::new(), host_phase_1: None, trampoline_phase_1: None, trampoline_phase_2: None, @@ -309,10 +309,6 @@ impl<'a> UpdatePlanBuilder<'a> { | KnownArtifactKind::SwitchSp => unreachable!(), }; - if rot_a.is_some() || rot_b.is_some() { - return Err(RepositoryError::DuplicateArtifactKind(artifact_kind)); - } - let (rot_a_data, rot_b_data) = Self::extract_nested_artifact_pair( &mut self.extracted_artifacts, artifact_kind, @@ -336,10 +332,8 @@ impl<'a> UpdatePlanBuilder<'a> { kind: rot_b_kind.clone(), }; - *rot_a = - Some(ArtifactIdData { id: rot_a_id, data: rot_a_data.clone() }); - *rot_b = - Some(ArtifactIdData { id: rot_b_id, data: rot_b_data.clone() }); + rot_a.push(ArtifactIdData { id: rot_a_id, data: rot_a_data.clone() }); + rot_b.push(ArtifactIdData { id: rot_b_id, data: rot_b_data.clone() }); record_extracted_artifact( artifact_id.clone(), @@ -574,53 +568,39 @@ impl<'a> UpdatePlanBuilder<'a> { pub(super) fn build(self) -> Result { // Ensure our multi-board-supporting kinds have at least one board // present. - if self.gimlet_sp.is_empty() { - return Err(RepositoryError::MissingArtifactKind( - KnownArtifactKind::GimletSp, - )); - } - if self.psc_sp.is_empty() { - return Err(RepositoryError::MissingArtifactKind( - KnownArtifactKind::PscSp, - )); - } - if self.sidecar_sp.is_empty() { - return Err(RepositoryError::MissingArtifactKind( - KnownArtifactKind::SwitchSp, - )); + for (kind, no_artifacts) in [ + (KnownArtifactKind::GimletSp, self.gimlet_sp.is_empty()), + (KnownArtifactKind::PscSp, self.psc_sp.is_empty()), + (KnownArtifactKind::SwitchSp, self.sidecar_sp.is_empty()), + ( + KnownArtifactKind::GimletRot, + self.gimlet_rot_a.is_empty() || self.gimlet_rot_b.is_empty(), + ), + ( + KnownArtifactKind::PscRot, + self.psc_rot_a.is_empty() || self.psc_rot_b.is_empty(), + ), + ( + KnownArtifactKind::SwitchRot, + self.sidecar_rot_a.is_empty() || self.sidecar_rot_b.is_empty(), + ), + ] { + if no_artifacts { + return Err(RepositoryError::MissingArtifactKind(kind)); + } } Ok(UpdatePlan { system_version: self.system_version, gimlet_sp: self.gimlet_sp, // checked above - gimlet_rot_a: self.gimlet_rot_a.ok_or( - RepositoryError::MissingArtifactKind( - KnownArtifactKind::GimletRot, - ), - )?, - gimlet_rot_b: self.gimlet_rot_b.ok_or( - RepositoryError::MissingArtifactKind( - KnownArtifactKind::GimletRot, - ), - )?, - psc_sp: self.psc_sp, // checked above - psc_rot_a: self.psc_rot_a.ok_or( - RepositoryError::MissingArtifactKind(KnownArtifactKind::PscRot), - )?, - psc_rot_b: self.psc_rot_b.ok_or( - RepositoryError::MissingArtifactKind(KnownArtifactKind::PscRot), - )?, + gimlet_rot_a: self.gimlet_rot_a, // checked above + gimlet_rot_b: self.gimlet_rot_b, // checked above + psc_sp: self.psc_sp, // checked above + psc_rot_a: self.psc_rot_a, // checked above + psc_rot_b: self.psc_rot_b, // checked above sidecar_sp: self.sidecar_sp, // checked above - sidecar_rot_a: self.sidecar_rot_a.ok_or( - RepositoryError::MissingArtifactKind( - KnownArtifactKind::SwitchRot, - ), - )?, - sidecar_rot_b: self.sidecar_rot_b.ok_or( - RepositoryError::MissingArtifactKind( - KnownArtifactKind::SwitchRot, - ), - )?, + sidecar_rot_a: self.sidecar_rot_a, // checked above + sidecar_rot_b: self.sidecar_rot_b, // checked above host_phase_1: self.host_phase_1.ok_or( RepositoryError::MissingArtifactKind(KnownArtifactKind::Host), )?, @@ -1030,21 +1010,27 @@ mod tests { // Check extracted RoT data assert_eq!( - read_to_vec(&plan.gimlet_rot_a.data).await, + read_to_vec(&plan.gimlet_rot_a[0].data).await, gimlet_rot.archive_a ); assert_eq!( - read_to_vec(&plan.gimlet_rot_b.data).await, + read_to_vec(&plan.gimlet_rot_b[0].data).await, gimlet_rot.archive_b ); - assert_eq!(read_to_vec(&plan.psc_rot_a.data).await, psc_rot.archive_a); - assert_eq!(read_to_vec(&plan.psc_rot_b.data).await, psc_rot.archive_b); assert_eq!( - read_to_vec(&plan.sidecar_rot_a.data).await, + read_to_vec(&plan.psc_rot_a[0].data).await, + psc_rot.archive_a + ); + assert_eq!( + read_to_vec(&plan.psc_rot_b[0].data).await, + psc_rot.archive_b + ); + assert_eq!( + read_to_vec(&plan.sidecar_rot_a[0].data).await, sidecar_rot.archive_a ); assert_eq!( - read_to_vec(&plan.sidecar_rot_b.data).await, + read_to_vec(&plan.sidecar_rot_b[0].data).await, sidecar_rot.archive_b ); diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 1bbda00158..e968d65a30 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -18,18 +18,23 @@ use anyhow::anyhow; use anyhow::bail; use anyhow::ensure; use anyhow::Context; +use base64::Engine; use display_error_chain::DisplayErrorChain; use dropshot::HttpError; +use futures::TryFutureExt; use gateway_client::types::HostPhase2Progress; use gateway_client::types::HostPhase2RecoveryImageId; use gateway_client::types::HostStartupOptions; use gateway_client::types::InstallinatorImageId; use gateway_client::types::PowerState; +use gateway_client::types::RotCfpaSlot; use gateway_client::types::SpComponentFirmwareSlot; use gateway_client::types::SpIdentifier; use gateway_client::types::SpType; use gateway_client::types::SpUpdateStatus; use gateway_messages::SpComponent; +use gateway_messages::ROT_PAGE_SIZE; +use hubtools::RawHubrisArchive; use installinator_common::InstallinatorCompletionMetadata; use installinator_common::InstallinatorSpec; use installinator_common::M2Slot; @@ -52,11 +57,13 @@ use std::sync::Mutex as StdMutex; use std::time::Duration; use std::time::Instant; use thiserror::Error; +use tokio::io::AsyncReadExt; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::sync::watch; use tokio::sync::Mutex; use tokio::task::JoinHandle; +use tokio_util::io::StreamReader; use update_engine::events::ProgressUnits; use update_engine::AbortHandle; use update_engine::StepSpec; @@ -768,19 +775,13 @@ impl UpdateDriver { } let (rot_a, rot_b, sp_artifacts) = match update_cx.sp.type_ { - SpType::Sled => ( - plan.gimlet_rot_a.clone(), - plan.gimlet_rot_b.clone(), - &plan.gimlet_sp, - ), - SpType::Power => { - (plan.psc_rot_a.clone(), plan.psc_rot_b.clone(), &plan.psc_sp) + SpType::Sled => { + (&plan.gimlet_rot_a, &plan.gimlet_rot_b, &plan.gimlet_sp) + } + SpType::Power => (&plan.psc_rot_a, &plan.psc_rot_b, &plan.psc_sp), + SpType::Switch => { + (&plan.sidecar_rot_a, &plan.sidecar_rot_b, &plan.sidecar_sp) } - SpType::Switch => ( - plan.sidecar_rot_a.clone(), - plan.sidecar_rot_b.clone(), - &plan.sidecar_sp, - ), }; let rot_registrar = engine.for_component(UpdateComponent::Rot); @@ -790,16 +791,15 @@ impl UpdateDriver { // currently executing; we must update the _other_ slot. We also want to // know its current version (so we can skip updating if we only need to // update the SP and/or host). - let rot_interrogation = - rot_registrar - .new_step( - UpdateStepId::InterrogateRot, - "Checking current RoT version and active slot", - |_cx| async move { - update_cx.interrogate_rot(rot_a, rot_b).await - }, - ) - .register(); + let rot_interrogation = rot_registrar + .new_step( + UpdateStepId::InterrogateRot, + "Checking current RoT version and active slot", + move |_cx| async move { + update_cx.interrogate_rot(rot_a, rot_b).await + }, + ) + .register(); // The SP only has one updateable firmware slot ("the inactive bank"). // We want to ask about slot 0 (the active slot)'s current version, and @@ -1557,8 +1557,8 @@ impl UpdateContext { async fn interrogate_rot( &self, - rot_a: ArtifactIdData, - rot_b: ArtifactIdData, + rot_a: &[ArtifactIdData], + rot_b: &[ArtifactIdData], ) -> Result, UpdateTerminalError> { let rot_active_slot = self .get_component_active_slot(SpComponent::ROT.const_as_str()) @@ -1569,7 +1569,7 @@ impl UpdateContext { // Flip these around: if 0 (A) is active, we want to // update 1 (B), and vice versa. - let (active_slot_name, slot_to_update, artifact_to_apply) = + let (active_slot_name, slot_to_update, available_artifacts) = match rot_active_slot { 0 => ('A', 1, rot_b), 1 => ('B', 0, rot_a), @@ -1582,6 +1582,127 @@ impl UpdateContext { } }; + // Read the CMPA and currently-active CFPA so we can find the + // correctly-signed artifact. + let base64_decode_rot_page = |data: String| { + // Even though we know `data` should decode to exactly + // `ROT_PAGE_SIZE` bytes, the base64 crate requires an output buffer + // of at least `decoded_len_estimate`. Allocate such a buffer here, + // then we'll copy to the fixed-size array we need after confirming + // the number of decoded bytes; + let mut output_buf = + vec![0; base64::decoded_len_estimate(data.len())]; + + let n = base64::engine::general_purpose::STANDARD + .decode_slice(&data, &mut output_buf) + .with_context(|| { + format!("failed to decode base64 string: {data:?}") + })?; + if n != ROT_PAGE_SIZE { + bail!( + "incorrect len ({n}, expected {ROT_PAGE_SIZE}) \ + after decoding base64 string: {data:?}", + ); + } + let mut page = [0; ROT_PAGE_SIZE]; + page.copy_from_slice(&output_buf[..n]); + Ok(page) + }; + let cmpa = self + .mgs_client + .sp_rot_cmpa_get( + self.sp.type_, + self.sp.slot, + SpComponent::ROT.const_as_str(), + ) + .await + .map_err(|err| UpdateTerminalError::GetRotCmpaFailed { + error: err.into(), + }) + .and_then(|response| { + let data = response.into_inner().base64_data; + base64_decode_rot_page(data).map_err(|error| { + UpdateTerminalError::GetRotCmpaFailed { error } + }) + })?; + let cfpa = self + .mgs_client + .sp_rot_cfpa_get( + self.sp.type_, + self.sp.slot, + SpComponent::ROT.const_as_str(), + &gateway_client::types::GetCfpaParams { + slot: RotCfpaSlot::Active, + }, + ) + .await + .map_err(|err| UpdateTerminalError::GetRotCfpaFailed { + error: err.into(), + }) + .and_then(|response| { + let data = response.into_inner().base64_data; + base64_decode_rot_page(data).map_err(|error| { + UpdateTerminalError::GetRotCfpaFailed { error } + }) + })?; + + // Loop through our possible artifacts and find the first (we only + // expect one!) that verifies against the RoT's CMPA/CFPA. + let mut artifact_to_apply = None; + for artifact in available_artifacts { + let image = artifact + .data + .reader_stream() + .and_then(|stream| async { + let mut buf = Vec::with_capacity(artifact.data.file_size()); + StreamReader::new(stream) + .read_to_end(&mut buf) + .await + .context("I/O error reading extracted archive")?; + Ok(buf) + }) + .await + .map_err(|error| { + UpdateTerminalError::FailedFindingSignedRotImage { error } + })?; + let archive = RawHubrisArchive::from_vec(image).map_err(|err| { + UpdateTerminalError::FailedFindingSignedRotImage { + error: anyhow::Error::new(err).context(format!( + "failed to read hubris archive for {:?}", + artifact.id + )), + } + })?; + match archive.verify(&cmpa, &cfpa) { + Ok(()) => { + info!( + self.log, "RoT archive verification success"; + "name" => artifact.id.name.as_str(), + "version" => %artifact.id.version, + "kind" => ?artifact.id.kind, + ); + artifact_to_apply = Some(artifact.clone()); + break; + } + Err(err) => { + // We log this but don't fail - we want to continue looking + // for a verifiable artifact. + info!( + self.log, "RoT archive verification failed"; + "artifact" => ?artifact.id, + "err" => %DisplayErrorChain::new(&err), + ); + } + } + } + + // Ensure we found a valid RoT artifact. + let artifact_to_apply = artifact_to_apply.ok_or_else(|| { + UpdateTerminalError::FailedFindingSignedRotImage { + error: anyhow!("no RoT image found with valid CMPA/CFPA"), + } + })?; + // Read the caboose of the currently-active slot. let caboose = self .mgs_client diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs index a198068ef3..af743190c2 100644 --- a/wicketd/tests/integration_tests/updates.rs +++ b/wicketd/tests/integration_tests/updates.rs @@ -169,8 +169,8 @@ async fn test_updates() { StepEventKind::ExecutionFailed { failed_step, .. } => { // TODO: obviously we shouldn't stop here, get past more of the // update process in this test. We currently fail when attempting to - // look up the SP's board in our tuf repo. - assert_eq!(failed_step.info.component, UpdateComponent::Sp); + // look up the RoT's CMPA/CFPA. + assert_eq!(failed_step.info.component, UpdateComponent::Rot); } other => { panic!("unexpected terminal event kind: {other:?}"); diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 36c3fe3f5f..b08f2612f1 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -43,7 +43,7 @@ futures-io = { version = "0.3.28", default-features = false, features = ["std"] futures-sink = { version = "0.3.28" } futures-task = { version = "0.3.28", default-features = false, features = ["std"] } futures-util = { version = "0.3.28", features = ["channel", "io", "sink"] } -gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", features = ["std"] } +gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } @@ -135,7 +135,7 @@ futures-io = { version = "0.3.28", default-features = false, features = ["std"] futures-sink = { version = "0.3.28" } futures-task = { version = "0.3.28", default-features = false, features = ["std"] } futures-util = { version = "0.3.28", features = ["channel", "io", "sink"] } -gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", features = ["std"] } +gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } From 1beda0b8399a302c0efa1af1226a649180b01bd9 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Thu, 19 Oct 2023 13:20:09 -0700 Subject: [PATCH 66/89] test-utils: use a per-user temp crdb seed dir (#4300) Make crdb seed generation for tests more amenable to running in shared environments. --- .github/buildomat/build-and-test.sh | 2 +- test-utils/src/dev/seed.rs | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index 8791ea2fa6..d8de288239 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -69,7 +69,7 @@ ptime -m timeout 1h cargo test --doc --locked --verbose --no-fail-fast # We expect the seed CRDB to be placed here, so we explicitly remove it so the # rmdir check below doesn't get triggered. Nextest doesn't have support for # teardown scripts so this is the best we've got. -rm -rf "$TEST_TMPDIR/crdb-base" +rm -rf "$TEST_TMPDIR/crdb-base"* # # Make sure that we have left nothing around in $TEST_TMPDIR. The easiest way diff --git a/test-utils/src/dev/seed.rs b/test-utils/src/dev/seed.rs index 841ecd5f35..7a75d0bbd3 100644 --- a/test-utils/src/dev/seed.rs +++ b/test-utils/src/dev/seed.rs @@ -92,12 +92,14 @@ pub async fn ensure_seed_tarball_exists( ); } - // XXX: we aren't considering cross-user permissions for this file. Might be - // worth setting more restrictive permissions on it, or using a per-user - // cache dir. + // If possible, try for a per-user folder in the temp dir + // to avoid clashes on shared build environments. + let crdb_base = std::env::var("USER") + .map(|user| format!("crdb-base-{user}")) + .unwrap_or("crdb-base".into()); let base_seed_dir = Utf8PathBuf::from_path_buf(std::env::temp_dir()) .expect("Not a UTF-8 path") - .join("crdb-base"); + .join(crdb_base); std::fs::create_dir_all(&base_seed_dir).unwrap(); let mut desired_seed_tar = base_seed_dir.join(digest_unique_to_schema()); desired_seed_tar.set_extension("tar"); From 36e047091a0793ae1d3a5f9bb176a2f7de137f5a Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Fri, 20 Oct 2023 10:23:45 -0400 Subject: [PATCH 67/89] Decrease Crucible resources in a CTE (#4290) `decrease_crucible_resource_count_and_soft_delete_volume` is currently implemented as a rather heavy weight transaction. This commit changes that to a CTE. What enabled this was learning that Cockroach can emit JSON through some built-in functions: part of what made the transaction heavy weight was that it was serializing the Crucible resources to clean up and recording that in a column for the soft-deleted Volume. The fact that this can occur in Cockroach means that the entire transaction can be converted into a CTE. --- nexus/db-queries/src/db/datastore/volume.rs | 412 ++++++++------- nexus/db-queries/src/db/queries/mod.rs | 1 + nexus/db-queries/src/db/queries/volume.rs | 497 ++++++++++++++++++ nexus/src/app/sagas/volume_delete.rs | 181 +++---- nexus/tests/integration_tests/snapshots.rs | 56 +- .../integration_tests/volume_management.rs | 30 +- 6 files changed, 822 insertions(+), 355 deletions(-) create mode 100644 nexus/db-queries/src/db/queries/volume.rs diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index f9f982213f..5d753f0742 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -14,10 +14,10 @@ use crate::db::model::Dataset; use crate::db::model::Region; use crate::db::model::RegionSnapshot; use crate::db::model::Volume; +use crate::db::queries::volume::DecreaseCrucibleResourceCountAndSoftDeleteVolume; use anyhow::bail; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use chrono::Utc; use diesel::prelude::*; use diesel::OptionalExtension; use omicron_common::api::external::CreateResult; @@ -27,6 +27,7 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::ResourceType; use serde::Deserialize; +use serde::Deserializer; use serde::Serialize; use sled_agent_client::types::VolumeConstructionRequest; use uuid::Uuid; @@ -533,16 +534,6 @@ impl DataStore { &self, volume_id: Uuid, ) -> Result { - #[derive(Debug, thiserror::Error)] - enum DecreaseCrucibleResourcesError { - #[error("Error during decrease Crucible resources: {0}")] - DieselError(#[from] diesel::result::Error), - - #[error("Serde error during decrease Crucible resources: {0}")] - SerdeError(#[from] serde_json::Error), - } - type TxnError = TransactionError; - // Grab all the targets that the volume construction request references. // Do this outside the transaction, as the data inside volume doesn't // change and this would simply add to the transaction time. @@ -574,12 +565,13 @@ impl DataStore { crucible_targets }; - // In a transaction: + // Call a CTE that will: // // 1. decrease the number of references for each region snapshot that // this Volume references // 2. soft-delete the volume - // 3. record the resources to clean up + // 3. record the resources to clean up as a serialized CrucibleResources + // struct in volume's `resources_to_clean_up` column. // // Step 3 is important because this function is called from a saga node. // If saga execution crashes after steps 1 and 2, but before serializing @@ -588,197 +580,48 @@ impl DataStore { // // We also have to guard against the case where this function is called // multiple times, and that is done by soft-deleting the volume during - // the transaction, and returning the previously serialized list of - // resources to clean up if a soft-delete has already occurred. - - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - // Grab the volume in question. If the volume record was already - // hard-deleted, assume clean-up has occurred and return an empty - // CrucibleResources. If the volume record was soft-deleted, then - // return the serialized CrucibleResources. - use db::schema::volume::dsl as volume_dsl; + // the CTE, and returning the previously serialized list of resources to + // clean up if a soft-delete has already occurred. - { - let volume = volume_dsl::volume - .filter(volume_dsl::id.eq(volume_id)) - .select(Volume::as_select()) - .get_result_async(&conn) - .await - .optional()?; - - let volume = if let Some(v) = volume { - v - } else { - // the volume was hard-deleted, return an empty - // CrucibleResources - return Ok(CrucibleResources::V1( - CrucibleResourcesV1::default(), - )); - }; + let _old_volume: Vec = + DecreaseCrucibleResourceCountAndSoftDeleteVolume::new( + volume_id, + crucible_targets.read_only_targets.clone(), + ) + .get_results_async::( + &*self.pool_connection_unauthorized().await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; - if volume.time_deleted.is_none() { - // a volume record exists, and was not deleted - this is the - // first time through this transaction for a certain volume - // id. Get the volume for use in the transaction. - volume - } else { - // this volume was soft deleted - this is a repeat time - // through this transaction. - - if let Some(resources_to_clean_up) = - volume.resources_to_clean_up - { - // return the serialized CrucibleResources - return serde_json::from_str( - &resources_to_clean_up, - ) - .map_err(|e| { - TxnError::CustomError( - DecreaseCrucibleResourcesError::SerdeError( - e, - ), - ) - }); - } else { - // If no CrucibleResources struct was serialized, that's - // definitely a bug of some sort - the soft-delete below - // sets time_deleted at the same time as - // resources_to_clean_up! But, instead of a panic here, - // just return an empty CrucibleResources. - return Ok(CrucibleResources::V1( - CrucibleResourcesV1::default(), - )); - } + // Get the updated Volume to get the resources to clean up + let resources_to_clean_up: CrucibleResources = match self + .volume_get(volume_id) + .await? + { + Some(volume) => { + match volume.resources_to_clean_up.as_ref() { + Some(v) => serde_json::from_str(v)?, + + None => { + // Even volumes with nothing to clean up should have + // a serialized CrucibleResources that contains + // empty vectors instead of None. Instead of + // panicing here though, just return the default + // (nothing to clean up). + CrucibleResources::V1(CrucibleResourcesV1::default()) } - }; - - // Decrease the number of uses for each non-deleted referenced - // region snapshot. - - use db::schema::region_snapshot::dsl; - - diesel::update(dsl::region_snapshot) - .filter( - dsl::snapshot_addr - .eq_any(crucible_targets.read_only_targets.clone()), - ) - .filter(dsl::volume_references.gt(0)) - .filter(dsl::deleting.eq(false)) - .set(dsl::volume_references.eq(dsl::volume_references - 1)) - .execute_async(&conn) - .await?; - - // Then, note anything that was set to zero from the above - // UPDATE, and then mark all those as deleted. - let snapshots_to_delete: Vec = - dsl::region_snapshot - .filter( - dsl::snapshot_addr.eq_any( - crucible_targets.read_only_targets.clone(), - ), - ) - .filter(dsl::volume_references.eq(0)) - .filter(dsl::deleting.eq(false)) - .select(RegionSnapshot::as_select()) - .load_async(&conn) - .await?; - - diesel::update(dsl::region_snapshot) - .filter( - dsl::snapshot_addr - .eq_any(crucible_targets.read_only_targets.clone()), - ) - .filter(dsl::volume_references.eq(0)) - .filter(dsl::deleting.eq(false)) - .set(dsl::deleting.eq(true)) - .execute_async(&conn) - .await?; - - // Return what results can be cleaned up - let result = CrucibleResources::V2(CrucibleResourcesV2 { - // The only use of a read-write region will be at the top level of a - // Volume. These are not shared, but if any snapshots are taken this - // will prevent deletion of the region. Filter out any regions that - // have associated snapshots. - datasets_and_regions: { - use db::schema::dataset::dsl as dataset_dsl; - use db::schema::region::dsl as region_dsl; - - // Return all regions for this volume_id, where there either are - // no region_snapshots, or region_snapshots.volume_references = - // 0. - region_dsl::region - .filter(region_dsl::volume_id.eq(volume_id)) - .inner_join( - dataset_dsl::dataset - .on(region_dsl::dataset_id - .eq(dataset_dsl::id)), - ) - .left_join( - dsl::region_snapshot.on(dsl::region_id - .eq(region_dsl::id) - .and(dsl::dataset_id.eq(dataset_dsl::id))), - ) - .filter( - dsl::volume_references - .eq(0) - // Despite the SQL specifying that this column is NOT NULL, - // this null check is required for this function to work! - // The left join of region_snapshot might cause a null here. - .or(dsl::volume_references.is_null()), - ) - .select((Dataset::as_select(), Region::as_select())) - .get_results_async::<(Dataset, Region)>(&conn) - .await? - }, - - // Consumers of this struct will be responsible for deleting - // the read-only downstairs running for the snapshot and the - // snapshot itself. - // - // It's important to not return *every* region snapshot with - // zero references: multiple volume delete sub-sagas will - // then be issues duplicate DELETE calls to Crucible agents, - // and a request to delete a read-only downstairs running - // for a snapshot that doesn't exist will return a 404, - // causing the saga to error and unwind. - snapshots_to_delete, - }); - - // Soft delete this volume, and serialize the resources that are to - // be cleaned up. - let now = Utc::now(); - diesel::update(volume_dsl::volume) - .filter(volume_dsl::id.eq(volume_id)) - .set(( - volume_dsl::time_deleted.eq(now), - volume_dsl::resources_to_clean_up.eq( - serde_json::to_string(&result).map_err(|e| { - TxnError::CustomError( - DecreaseCrucibleResourcesError::SerdeError( - e, - ), - ) - })?, - ), - )) - .execute_async(&conn) - .await?; + } + } - Ok(result) - }) - .await - .map_err(|e| match e { - TxnError::CustomError( - DecreaseCrucibleResourcesError::DieselError(e), - ) => public_error_from_diesel(e, ErrorHandler::Server), + None => { + // If the volume was hard-deleted already, return the + // default (nothing to clean up). + CrucibleResources::V1(CrucibleResourcesV1::default()) + } + }; - _ => { - Error::internal_error(&format!("Transaction error: {}", e)) - } - }) + Ok(resources_to_clean_up) } // Here we remove the read only parent from volume_id, and attach it @@ -993,6 +836,7 @@ pub struct CrucibleTargets { pub enum CrucibleResources { V1(CrucibleResourcesV1), V2(CrucibleResourcesV2), + V3(CrucibleResourcesV3), } #[derive(Debug, Default, Serialize, Deserialize)] @@ -1007,6 +851,176 @@ pub struct CrucibleResourcesV2 { pub snapshots_to_delete: Vec, } +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct RegionSnapshotV3 { + dataset: Uuid, + region: Uuid, + snapshot: Uuid, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct CrucibleResourcesV3 { + #[serde(deserialize_with = "null_to_empty_list")] + pub regions: Vec, + + #[serde(deserialize_with = "null_to_empty_list")] + pub region_snapshots: Vec, +} + +// Cockroach's `json_agg` will emit a `null` instead of a `[]` if a SELECT +// returns zero rows. Handle that with this function when deserializing. +fn null_to_empty_list<'de, D, T>(de: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + Ok(match Option::>::deserialize(de)? { + Some(v) => v, + None => vec![], + }) +} + +impl DataStore { + /// For a CrucibleResources object, return the Regions to delete, as well as + /// the Dataset they belong to. + pub async fn regions_to_delete( + &self, + crucible_resources: &CrucibleResources, + ) -> LookupResult> { + let conn = self.pool_connection_unauthorized().await?; + + match crucible_resources { + CrucibleResources::V1(crucible_resources) => { + Ok(crucible_resources.datasets_and_regions.clone()) + } + + CrucibleResources::V2(crucible_resources) => { + Ok(crucible_resources.datasets_and_regions.clone()) + } + + CrucibleResources::V3(crucible_resources) => { + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region::dsl as region_dsl; + + region_dsl::region + .filter( + region_dsl::id + .eq_any(crucible_resources.regions.clone()), + ) + .inner_join( + dataset_dsl::dataset + .on(region_dsl::dataset_id.eq(dataset_dsl::id)), + ) + .select((Dataset::as_select(), Region::as_select())) + .get_results_async::<(Dataset, Region)>(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + } + } + } + + /// For a CrucibleResources object, return the RegionSnapshots to delete, as + /// well as the Dataset they belong to. + pub async fn snapshots_to_delete( + &self, + crucible_resources: &CrucibleResources, + ) -> LookupResult> { + let conn = self.pool_connection_unauthorized().await?; + + match crucible_resources { + CrucibleResources::V1(crucible_resources) => { + Ok(crucible_resources.datasets_and_snapshots.clone()) + } + + CrucibleResources::V2(crucible_resources) => { + use db::schema::dataset::dsl; + + let mut result: Vec<_> = Vec::with_capacity( + crucible_resources.snapshots_to_delete.len(), + ); + + for snapshots_to_delete in + &crucible_resources.snapshots_to_delete + { + let maybe_dataset = dsl::dataset + .filter(dsl::id.eq(snapshots_to_delete.dataset_id)) + .select(Dataset::as_select()) + .first_async(&*conn) + .await + .optional() + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + match maybe_dataset { + Some(dataset) => { + result.push((dataset, snapshots_to_delete.clone())); + } + + None => { + return Err(Error::internal_error(&format!( + "could not find dataset {}!", + snapshots_to_delete.dataset_id, + ))); + } + } + } + + Ok(result) + } + + CrucibleResources::V3(crucible_resources) => { + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region_snapshot::dsl; + + let mut datasets_and_snapshots = Vec::with_capacity( + crucible_resources.region_snapshots.len(), + ); + + for region_snapshots in &crucible_resources.region_snapshots { + let maybe_tuple = dsl::region_snapshot + .filter(dsl::dataset_id.eq(region_snapshots.dataset)) + .filter(dsl::region_id.eq(region_snapshots.region)) + .filter(dsl::snapshot_id.eq(region_snapshots.snapshot)) + .inner_join( + dataset_dsl::dataset + .on(dsl::dataset_id.eq(dataset_dsl::id)), + ) + .select(( + Dataset::as_select(), + RegionSnapshot::as_select(), + )) + .first_async::<(Dataset, RegionSnapshot)>(&*conn) + .await + .optional() + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + match maybe_tuple { + Some(tuple) => { + datasets_and_snapshots.push(tuple); + } + + None => { + // If something else is deleting the exact same + // CrucibleResources (for example from a duplicate + // resource-delete saga) then these region_snapshot + // entries could be gone (because they are hard + // deleted). Skip missing entries, return only what + // we can find. + } + } + } + + Ok(datasets_and_snapshots) + } + } + } +} + /// Return the targets from a VolumeConstructionRequest. /// /// The targets of a volume construction request map to resources. diff --git a/nexus/db-queries/src/db/queries/mod.rs b/nexus/db-queries/src/db/queries/mod.rs index cd48be61e3..a1022f9187 100644 --- a/nexus/db-queries/src/db/queries/mod.rs +++ b/nexus/db-queries/src/db/queries/mod.rs @@ -14,6 +14,7 @@ mod next_item; pub mod network_interface; pub mod region_allocation; pub mod virtual_provisioning_collection_update; +pub mod volume; pub mod vpc; pub mod vpc_subnet; diff --git a/nexus/db-queries/src/db/queries/volume.rs b/nexus/db-queries/src/db/queries/volume.rs new file mode 100644 index 0000000000..31882dca89 --- /dev/null +++ b/nexus/db-queries/src/db/queries/volume.rs @@ -0,0 +1,497 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Helper queries for working with volumes. + +use crate::db; +use crate::db::pool::DbConnection; +use diesel::expression::is_aggregate; +use diesel::expression::ValidGrouping; +use diesel::pg::Pg; +use diesel::query_builder::AstPass; +use diesel::query_builder::Query; +use diesel::query_builder::QueryFragment; +use diesel::query_builder::QueryId; +use diesel::sql_types; +use diesel::Column; +use diesel::Expression; +use diesel::QueryResult; +use diesel::RunQueryDsl; +use uuid::Uuid; + +/// Produces a query fragment that will act as a filter for the data modifying +/// sub-queries of the "decrease crucible resource count and soft delete volume" CTE. +/// +/// The output should look like: +/// +/// ```sql +/// (SELECT CASE +/// WHEN volume.resources_to_clean_up is null then true +/// ELSE false +/// END +/// FROM volume WHERE id = '{}') +/// ``` +#[must_use = "Queries must be executed"] +struct ResourcesToCleanUpColumnIsNull { + volume_id: Uuid, +} + +impl ResourcesToCleanUpColumnIsNull { + pub fn new(volume_id: Uuid) -> Self { + Self { volume_id } + } +} + +impl QueryId for ResourcesToCleanUpColumnIsNull { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for ResourcesToCleanUpColumnIsNull { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::volume; + + out.push_sql("SELECT CASE WHEN "); + out.push_identifier(volume::dsl::resources_to_clean_up::NAME)?; + out.push_sql(" is null then true ELSE false END FROM "); + volume::dsl::volume.walk_ast(out.reborrow())?; + out.push_sql(" WHERE "); + out.push_identifier(volume::dsl::id::NAME)?; + out.push_sql(" = "); + out.push_bind_param::(&self.volume_id)?; + + Ok(()) + } +} + +impl Expression for ResourcesToCleanUpColumnIsNull { + type SqlType = sql_types::Bool; +} + +impl ValidGrouping + for ResourcesToCleanUpColumnIsNull +{ + type IsAggregate = is_aggregate::Never; +} + +/// Produces a query fragment that will conditionally reduce the volume +/// references for region_snapshot rows whose snapshot_addr column is part of a +/// list. +/// +/// The output should look like: +/// +/// ```sql +/// update region_snapshot set +/// volume_references = volume_references - 1, +/// deleting = case when volume_references = 1 +/// then true +/// else false +/// end +/// where +/// snapshot_addr in ('a1', 'a2', 'a3') and +/// volume_references >= 1 and +/// deleting = false and +/// () +/// returning * +/// ``` +#[must_use = "Queries must be executed"] +struct ConditionallyDecreaseReferences { + resources_to_clean_up_column_is_null_clause: ResourcesToCleanUpColumnIsNull, + snapshot_addrs: Vec, +} + +impl ConditionallyDecreaseReferences { + pub fn new(volume_id: Uuid, snapshot_addrs: Vec) -> Self { + Self { + resources_to_clean_up_column_is_null_clause: + ResourcesToCleanUpColumnIsNull::new(volume_id), + snapshot_addrs, + } + } +} + +impl QueryId for ConditionallyDecreaseReferences { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for ConditionallyDecreaseReferences { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::region_snapshot::dsl; + + out.push_sql("UPDATE "); + dsl::region_snapshot.walk_ast(out.reborrow())?; + out.push_sql(" SET "); + out.push_identifier(dsl::volume_references::NAME)?; + out.push_sql(" = "); + out.push_identifier(dsl::volume_references::NAME)?; + out.push_sql(" - 1, "); + out.push_identifier(dsl::deleting::NAME)?; + out.push_sql(" = CASE WHEN "); + out.push_identifier(dsl::volume_references::NAME)?; + out.push_sql(" = 1 THEN TRUE ELSE FALSE END WHERE "); + out.push_identifier(dsl::snapshot_addr::NAME)?; + out.push_sql(" IN ("); + + // If self.snapshot_addrs is empty, this query fragment will intentionally not update any + // region_snapshot rows. The rest of the CTE should still run to completion. + for (i, snapshot_addr) in self.snapshot_addrs.iter().enumerate() { + out.push_bind_param::(snapshot_addr)?; + if i == self.snapshot_addrs.len() - 1 { + out.push_sql(" "); + } else { + out.push_sql(", "); + } + } + + out.push_sql(") AND "); + out.push_identifier(dsl::volume_references::NAME)?; + out.push_sql(" >= 1 AND "); + out.push_identifier(dsl::deleting::NAME)?; + out.push_sql(" = false AND ( "); + self.resources_to_clean_up_column_is_null_clause + .walk_ast(out.reborrow())?; + out.push_sql(") RETURNING *"); + + Ok(()) + } +} + +impl Expression for ConditionallyDecreaseReferences { + type SqlType = sql_types::Array; +} + +impl ValidGrouping + for ConditionallyDecreaseReferences +{ + type IsAggregate = is_aggregate::Never; +} + +/// Produces a query fragment that will find all resources that can be cleaned +/// up as a result of a volume delete, and build a serialized JSON struct that +/// can be deserialized into a CrucibleResources::V3 variant. The output of this +/// will be written into the 'resources_to_clean_up` column of the volume being +/// soft-deleted. +/// +/// The output should look like: +/// +/// ```sql +/// json_build_object('V3', +/// json_build_object( +/// 'regions', (select json_agg(id) from region join t2 on region.id = t2.region_id where (t2.volume_references = 0 or t2.volume_references is null) and region.volume_id = ''), +/// 'region_snapshots', (select json_agg(json_build_object('dataset', dataset_id, 'region', region_id, 'snapshot', snapshot_id)) from t2 where t2.volume_references = 0) +/// ) +/// ) +/// ``` +/// +/// Note if json_agg is executing over zero rows, then the output is `null`, not +/// `[]`. For example, if the sub-query meant to return regions to clean up +/// returned zero rows, the output of json_build_object would look like: +/// +/// ```json +/// { +/// "V3": { +/// "regions": null, +/// ... +/// } +/// } +/// ``` +/// +/// Correctly handling `null` here is done in the deserializer for +/// CrucibleResourcesV3. +/// +/// A populated object should look like: +/// +/// ```json +/// { +/// "V3": { +/// "regions": [ +/// "9caae5bb-a212-4496-882a-af1ee242c62f", +/// "713c84ee-6b13-4301-b7a2-36debc7ee37e" +/// ], +/// "region_snapshots": [ +/// { +/// "dataset": "33ec5f07-5e7f-481e-966a-0fbc50d9ed3b", +/// "region": "1e2b1a75-9a58-4e5c-89a0-0cfd19ba055a", +/// "snapshot": "f7c8ed87-a67e-4d2b-8f35-3e8034de1c6f" +/// }, +/// { +/// "dataset": "5a16b1d6-7381-4c51-b49c-997624d43ead", +/// "region": "52b4c9bc-d1c9-4a3b-87c3-8e4501a883b0", +/// "snapshot": "2dd912e4-74db-409a-8d55-9795496cb320" +/// } +/// ] +/// } +/// } +/// ``` +#[must_use = "Queries must be executed"] +struct BuildJsonResourcesToCleanUp { + table: &'static str, + volume_id: Uuid, +} + +impl BuildJsonResourcesToCleanUp { + pub fn new(table: &'static str, volume_id: Uuid) -> Self { + Self { table, volume_id } + } +} + +impl QueryId for BuildJsonResourcesToCleanUp { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for BuildJsonResourcesToCleanUp { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::region::dsl as region_dsl; + use db::schema::region_snapshot::dsl as region_snapshot_dsl; + use db::schema::volume::dsl; + + out.push_sql("json_build_object('V3', "); + out.push_sql("json_build_object('regions', "); + out.push_sql("(SELECT json_agg("); + out.push_identifier(dsl::id::NAME)?; + out.push_sql(") FROM "); + region_dsl::region.walk_ast(out.reborrow())?; + out.push_sql(" JOIN "); + out.push_sql(self.table); + out.push_sql(" ON "); + out.push_identifier(region_dsl::id::NAME)?; + out.push_sql(" = "); + out.push_sql(self.table); + out.push_sql("."); + out.push_identifier(region_snapshot_dsl::region_id::NAME)?; // table's schema is equivalent to region_snapshot + out.push_sql(" WHERE ( "); + + out.push_sql(self.table); + out.push_sql("."); + out.push_identifier(region_snapshot_dsl::volume_references::NAME)?; + out.push_sql(" = 0 OR "); + out.push_sql(self.table); + out.push_sql("."); + out.push_identifier(region_snapshot_dsl::volume_references::NAME)?; + out.push_sql(" IS NULL"); + + out.push_sql(") AND "); + out.push_identifier(region_dsl::volume_id::NAME)?; + out.push_sql(" = "); + out.push_bind_param::(&self.volume_id)?; + + out.push_sql("), 'region_snapshots', ("); + out.push_sql("SELECT json_agg(json_build_object("); + out.push_sql("'dataset', "); + out.push_identifier(region_snapshot_dsl::dataset_id::NAME)?; + out.push_sql(", 'region', "); + out.push_identifier(region_snapshot_dsl::region_id::NAME)?; + out.push_sql(", 'snapshot', "); + out.push_identifier(region_snapshot_dsl::snapshot_id::NAME)?; + out.push_sql(")) from "); + out.push_sql(self.table); + out.push_sql(" where "); + out.push_sql(self.table); + out.push_sql("."); + out.push_identifier(region_snapshot_dsl::volume_references::NAME)?; + out.push_sql(" = 0)))"); + + Ok(()) + } +} + +impl ValidGrouping + for BuildJsonResourcesToCleanUp +{ + type IsAggregate = is_aggregate::Never; +} + +/// Produces a query fragment that will set the `resources_to_clean_up` column +/// of the volume being deleted if it is not set already. +/// +/// The output should look like: +/// +/// ```sql +/// update volume set +/// time_deleted = now(), +/// resources_to_clean_up = ( select ) +/// where id = '' and +/// () +/// returning volume.* +/// ``` +#[must_use = "Queries must be executed"] +struct ConditionallyUpdateVolume { + resources_to_clean_up_column_is_null_clause: ResourcesToCleanUpColumnIsNull, + build_json_resources_to_clean_up_query: BuildJsonResourcesToCleanUp, + volume_id: Uuid, +} + +impl ConditionallyUpdateVolume { + pub fn new(volume_id: Uuid, table: &'static str) -> Self { + Self { + resources_to_clean_up_column_is_null_clause: + ResourcesToCleanUpColumnIsNull::new(volume_id), + build_json_resources_to_clean_up_query: + BuildJsonResourcesToCleanUp::new(table, volume_id), + volume_id, + } + } +} + +impl QueryId for ConditionallyUpdateVolume { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for ConditionallyUpdateVolume { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::volume::dsl; + + out.push_sql("UPDATE "); + dsl::volume.walk_ast(out.reborrow())?; + out.push_sql(" SET "); + out.push_identifier(dsl::time_deleted::NAME)?; + out.push_sql(" = now(), "); + out.push_identifier(dsl::resources_to_clean_up::NAME)?; + out.push_sql(" = (SELECT "); + + self.build_json_resources_to_clean_up_query.walk_ast(out.reborrow())?; + + out.push_sql(") WHERE "); + out.push_identifier(dsl::id::NAME)?; + out.push_sql(" = "); + out.push_bind_param::(&self.volume_id)?; + out.push_sql(" AND ("); + + self.resources_to_clean_up_column_is_null_clause + .walk_ast(out.reborrow())?; + + out.push_sql(") RETURNING volume.*"); + + Ok(()) + } +} + +impl Expression for ConditionallyUpdateVolume { + type SqlType = diesel::sql_types::Array; +} + +impl ValidGrouping for ConditionallyUpdateVolume { + type IsAggregate = is_aggregate::Never; +} + +/// Produces a query fragment that will +/// +/// 1. decrease the number of references for each region snapshot that +/// a volume references +/// 2. soft-delete the volume +/// 3. record the resources to clean up as a serialized CrucibleResources +/// struct in volume's `resources_to_clean_up` column. +/// +/// The output should look like: +/// +/// ```sql +/// with UPDATED_REGION_SNAPSHOTS_TABLE as ( +/// UPDATE region_snapshot +/// ), +/// REGION_SNAPSHOTS_TO_CLEAN_UP_TABLE as ( +/// select * from UPDATED_REGION_SNAPSHOTS_TABLE where deleting = true and volume_references = 0 +/// ), +/// UPDATED_VOLUME_TABLE as ( +/// UPDATE volume +/// ) +/// select case +/// when volume.resources_to_clean_up is not null then volume.resources_to_clean_up +/// else (select resources_to_clean_up from UPDATED_VOLUME_TABLE where id = '') +/// end +/// from volume where id = ''; +/// ``` +#[must_use = "Queries must be executed"] +pub struct DecreaseCrucibleResourceCountAndSoftDeleteVolume { + conditionally_decrease_references: ConditionallyDecreaseReferences, + conditionally_update_volume_query: ConditionallyUpdateVolume, + volume_id: Uuid, +} + +impl DecreaseCrucibleResourceCountAndSoftDeleteVolume { + const UPDATED_REGION_SNAPSHOTS_TABLE: &str = "updated_region_snapshots"; + const REGION_SNAPSHOTS_TO_CLEAN_UP_TABLE: &str = + "region_snapshots_to_clean_up"; + const UPDATED_VOLUME_TABLE: &str = "updated_volume"; + + pub fn new(volume_id: Uuid, snapshot_addrs: Vec) -> Self { + Self { + conditionally_decrease_references: + ConditionallyDecreaseReferences::new(volume_id, snapshot_addrs), + conditionally_update_volume_query: ConditionallyUpdateVolume::new( + volume_id, + Self::REGION_SNAPSHOTS_TO_CLEAN_UP_TABLE, + ), + volume_id, + } + } +} + +impl QueryId for DecreaseCrucibleResourceCountAndSoftDeleteVolume { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for DecreaseCrucibleResourceCountAndSoftDeleteVolume { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::region_snapshot::dsl as rs_dsl; + use db::schema::volume::dsl; + + out.push_sql("WITH "); + out.push_sql(Self::UPDATED_REGION_SNAPSHOTS_TABLE); + out.push_sql(" as ("); + self.conditionally_decrease_references.walk_ast(out.reborrow())?; + out.push_sql("), "); + + out.push_sql(Self::REGION_SNAPSHOTS_TO_CLEAN_UP_TABLE); + out.push_sql(" AS (SELECT * FROM "); + out.push_sql(Self::UPDATED_REGION_SNAPSHOTS_TABLE); + out.push_sql(" WHERE "); + out.push_identifier(rs_dsl::deleting::NAME)?; + out.push_sql(" = TRUE AND "); + out.push_identifier(rs_dsl::volume_references::NAME)?; + out.push_sql(" = 0), "); + + out.push_sql(Self::UPDATED_VOLUME_TABLE); + out.push_sql(" AS ("); + self.conditionally_update_volume_query.walk_ast(out.reborrow())?; + out.push_sql(") "); + + out.push_sql("SELECT "); + dsl::volume.walk_ast(out.reborrow())?; + out.push_sql(".* FROM "); + dsl::volume.walk_ast(out.reborrow())?; + out.push_sql(" WHERE "); + out.push_identifier(dsl::id::NAME)?; + out.push_sql(" = "); + out.push_bind_param::(&self.volume_id)?; + + Ok(()) + } +} + +impl Expression for DecreaseCrucibleResourceCountAndSoftDeleteVolume { + type SqlType = diesel::sql_types::Array; +} + +impl ValidGrouping + for DecreaseCrucibleResourceCountAndSoftDeleteVolume +{ + type IsAggregate = is_aggregate::Never; +} + +impl RunQueryDsl + for DecreaseCrucibleResourceCountAndSoftDeleteVolume +{ +} + +type SelectableSql = < + >::SelectExpression as diesel::Expression +>::SqlType; + +impl Query for DecreaseCrucibleResourceCountAndSoftDeleteVolume { + type SqlType = SelectableSql; +} diff --git a/nexus/src/app/sagas/volume_delete.rs b/nexus/src/app/sagas/volume_delete.rs index d6358d5435..43530e913c 100644 --- a/nexus/src/app/sagas/volume_delete.rs +++ b/nexus/src/app/sagas/volume_delete.rs @@ -155,15 +155,19 @@ async fn svd_delete_crucible_regions( sagactx.lookup::("crucible_resources_to_delete")?; // Send DELETE calls to the corresponding Crucible agents - let datasets_and_regions = match crucible_resources_to_delete { - CrucibleResources::V1(crucible_resources_to_delete) => { - crucible_resources_to_delete.datasets_and_regions - } - - CrucibleResources::V2(crucible_resources_to_delete) => { - crucible_resources_to_delete.datasets_and_regions - } - }; + let datasets_and_regions = osagactx + .datastore() + .regions_to_delete( + &crucible_resources_to_delete, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to get datasets_and_regions from crucible resources ({:?}): {:?}", + crucible_resources_to_delete, + e, + )) + })?; delete_crucible_regions(log, datasets_and_regions.clone()).await.map_err( |e| { @@ -208,31 +212,19 @@ async fn svd_delete_crucible_running_snapshots( sagactx.lookup::("crucible_resources_to_delete")?; // Send DELETE calls to the corresponding Crucible agents - let datasets_and_snapshots = match crucible_resources_to_delete { - CrucibleResources::V1(crucible_resources_to_delete) => { - crucible_resources_to_delete.datasets_and_snapshots - } - - CrucibleResources::V2(crucible_resources_to_delete) => { - let mut datasets_and_snapshots: Vec<_> = Vec::with_capacity( - crucible_resources_to_delete.snapshots_to_delete.len(), - ); - - for region_snapshot in - crucible_resources_to_delete.snapshots_to_delete - { - let dataset = osagactx - .datastore() - .dataset_get(region_snapshot.dataset_id) - .await - .map_err(ActionError::action_failed)?; - - datasets_and_snapshots.push((dataset, region_snapshot)); - } - - datasets_and_snapshots - } - }; + let datasets_and_snapshots = osagactx + .datastore() + .snapshots_to_delete( + &crucible_resources_to_delete, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to get datasets_and_snapshots from crucible resources ({:?}): {:?}", + crucible_resources_to_delete, + e, + )) + })?; delete_crucible_running_snapshots(log, datasets_and_snapshots.clone()) .await @@ -261,31 +253,19 @@ async fn svd_delete_crucible_snapshots( sagactx.lookup::("crucible_resources_to_delete")?; // Send DELETE calls to the corresponding Crucible agents - let datasets_and_snapshots = match crucible_resources_to_delete { - CrucibleResources::V1(crucible_resources_to_delete) => { - crucible_resources_to_delete.datasets_and_snapshots - } - - CrucibleResources::V2(crucible_resources_to_delete) => { - let mut datasets_and_snapshots: Vec<_> = Vec::with_capacity( - crucible_resources_to_delete.snapshots_to_delete.len(), - ); - - for region_snapshot in - crucible_resources_to_delete.snapshots_to_delete - { - let dataset = osagactx - .datastore() - .dataset_get(region_snapshot.dataset_id) - .await - .map_err(ActionError::action_failed)?; - - datasets_and_snapshots.push((dataset, region_snapshot)); - } - - datasets_and_snapshots - } - }; + let datasets_and_snapshots = osagactx + .datastore() + .snapshots_to_delete( + &crucible_resources_to_delete, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to get datasets_and_snapshots from crucible resources ({:?}): {:?}", + crucible_resources_to_delete, + e, + )) + })?; delete_crucible_snapshots(log, datasets_and_snapshots.clone()) .await @@ -308,56 +288,39 @@ async fn svd_delete_crucible_snapshot_records( let crucible_resources_to_delete = sagactx.lookup::("crucible_resources_to_delete")?; - match crucible_resources_to_delete { - CrucibleResources::V1(crucible_resources_to_delete) => { - // Remove DB records - for (_, region_snapshot) in - &crucible_resources_to_delete.datasets_and_snapshots - { - osagactx - .datastore() - .region_snapshot_remove( - region_snapshot.dataset_id, - region_snapshot.region_id, - region_snapshot.snapshot_id, - ) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "failed to region_snapshot_remove {} {} {}: {:?}", - region_snapshot.dataset_id, - region_snapshot.region_id, - region_snapshot.snapshot_id, - e, - )) - })?; - } - } - - CrucibleResources::V2(crucible_resources_to_delete) => { - // Remove DB records - for region_snapshot in - &crucible_resources_to_delete.snapshots_to_delete - { - osagactx - .datastore() - .region_snapshot_remove( - region_snapshot.dataset_id, - region_snapshot.region_id, - region_snapshot.snapshot_id, - ) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "failed to region_snapshot_remove {} {} {}: {:?}", - region_snapshot.dataset_id, - region_snapshot.region_id, - region_snapshot.snapshot_id, - e, - )) - })?; - } - } + // Remove DB records + let datasets_and_snapshots = osagactx + .datastore() + .snapshots_to_delete( + &crucible_resources_to_delete, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to get datasets_and_snapshots from crucible resources ({:?}): {:?}", + crucible_resources_to_delete, + e, + )) + })?; + + for (_, region_snapshot) in datasets_and_snapshots { + osagactx + .datastore() + .region_snapshot_remove( + region_snapshot.dataset_id, + region_snapshot.region_id, + region_snapshot.snapshot_id, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to region_snapshot_remove {} {} {}: {:?}", + region_snapshot.dataset_id, + region_snapshot.region_id, + region_snapshot.snapshot_id, + e, + )) + })?; } Ok(()) diff --git a/nexus/tests/integration_tests/snapshots.rs b/nexus/tests/integration_tests/snapshots.rs index 68f4cdadd2..b3cf4bb594 100644 --- a/nexus/tests/integration_tests/snapshots.rs +++ b/nexus/tests/integration_tests/snapshots.rs @@ -1287,45 +1287,47 @@ async fn test_multiple_deletes_not_sent(cptestctx: &ControlPlaneTestContext) { .await .unwrap(); - let resources_1 = match resources_1 { - db::datastore::CrucibleResources::V1(_) => panic!("using old style!"), - db::datastore::CrucibleResources::V2(resources_1) => resources_1, - }; - let resources_2 = match resources_2 { - db::datastore::CrucibleResources::V1(_) => panic!("using old style!"), - db::datastore::CrucibleResources::V2(resources_2) => resources_2, - }; - let resources_3 = match resources_3 { - db::datastore::CrucibleResources::V1(_) => panic!("using old style!"), - db::datastore::CrucibleResources::V2(resources_3) => resources_3, - }; + let resources_1_datasets_and_regions = + datastore.regions_to_delete(&resources_1).await.unwrap(); + let resources_1_datasets_and_snapshots = + datastore.snapshots_to_delete(&resources_1).await.unwrap(); + + let resources_2_datasets_and_regions = + datastore.regions_to_delete(&resources_2).await.unwrap(); + let resources_2_datasets_and_snapshots = + datastore.snapshots_to_delete(&resources_2).await.unwrap(); + + let resources_3_datasets_and_regions = + datastore.regions_to_delete(&resources_3).await.unwrap(); + let resources_3_datasets_and_snapshots = + datastore.snapshots_to_delete(&resources_3).await.unwrap(); // No region deletions yet, these are just snapshot deletes - assert!(resources_1.datasets_and_regions.is_empty()); - assert!(resources_2.datasets_and_regions.is_empty()); - assert!(resources_3.datasets_and_regions.is_empty()); + assert!(resources_1_datasets_and_regions.is_empty()); + assert!(resources_2_datasets_and_regions.is_empty()); + assert!(resources_3_datasets_and_regions.is_empty()); // But there are snapshots to delete - assert!(!resources_1.snapshots_to_delete.is_empty()); - assert!(!resources_2.snapshots_to_delete.is_empty()); - assert!(!resources_3.snapshots_to_delete.is_empty()); + assert!(!resources_1_datasets_and_snapshots.is_empty()); + assert!(!resources_2_datasets_and_snapshots.is_empty()); + assert!(!resources_3_datasets_and_snapshots.is_empty()); // Assert there are no overlaps in the snapshots_to_delete to delete. - for tuple in &resources_1.snapshots_to_delete { - assert!(!resources_2.snapshots_to_delete.contains(tuple)); - assert!(!resources_3.snapshots_to_delete.contains(tuple)); + for tuple in &resources_1_datasets_and_snapshots { + assert!(!resources_2_datasets_and_snapshots.contains(tuple)); + assert!(!resources_3_datasets_and_snapshots.contains(tuple)); } - for tuple in &resources_2.snapshots_to_delete { - assert!(!resources_1.snapshots_to_delete.contains(tuple)); - assert!(!resources_3.snapshots_to_delete.contains(tuple)); + for tuple in &resources_2_datasets_and_snapshots { + assert!(!resources_1_datasets_and_snapshots.contains(tuple)); + assert!(!resources_3_datasets_and_snapshots.contains(tuple)); } - for tuple in &resources_3.snapshots_to_delete { - assert!(!resources_1.snapshots_to_delete.contains(tuple)); - assert!(!resources_2.snapshots_to_delete.contains(tuple)); + for tuple in &resources_3_datasets_and_snapshots { + assert!(!resources_1_datasets_and_snapshots.contains(tuple)); + assert!(!resources_2_datasets_and_snapshots.contains(tuple)); } } diff --git a/nexus/tests/integration_tests/volume_management.rs b/nexus/tests/integration_tests/volume_management.rs index f5935676a7..24a0e5591b 100644 --- a/nexus/tests/integration_tests/volume_management.rs +++ b/nexus/tests/integration_tests/volume_management.rs @@ -2219,17 +2219,12 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { assert_eq!(region_snapshot.deleting, true); } - match cr { - nexus_db_queries::db::datastore::CrucibleResources::V1(cr) => { - assert!(cr.datasets_and_regions.is_empty()); - assert_eq!(cr.datasets_and_snapshots.len(), 3); - } + let datasets_and_regions = datastore.regions_to_delete(&cr).await.unwrap(); + let datasets_and_snapshots = + datastore.snapshots_to_delete(&cr).await.unwrap(); - nexus_db_queries::db::datastore::CrucibleResources::V2(cr) => { - assert!(cr.datasets_and_regions.is_empty()); - assert_eq!(cr.snapshots_to_delete.len(), 3); - } - } + assert!(datasets_and_regions.is_empty()); + assert_eq!(datasets_and_snapshots.len(), 3); // Now, let's say we're at a spot where the running snapshots have been // deleted, but before volume_hard_delete or region_snapshot_remove are @@ -2350,17 +2345,12 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { assert_eq!(region_snapshot.deleting, true); } - match cr { - nexus_db_queries::db::datastore::CrucibleResources::V1(cr) => { - assert!(cr.datasets_and_regions.is_empty()); - assert_eq!(cr.datasets_and_snapshots.len(), 3); - } + let datasets_and_regions = datastore.regions_to_delete(&cr).await.unwrap(); + let datasets_and_snapshots = + datastore.snapshots_to_delete(&cr).await.unwrap(); - nexus_db_queries::db::datastore::CrucibleResources::V2(cr) => { - assert!(cr.datasets_and_regions.is_empty()); - assert_eq!(cr.snapshots_to_delete.len(), 3); - } - } + assert!(datasets_and_regions.is_empty()); + assert_eq!(datasets_and_snapshots.len(), 3); } #[nexus_test] From 9191af67b9aff710dac93ada6012b1e9b7d0c79c Mon Sep 17 00:00:00 2001 From: Greg Colombo Date: Fri, 20 Oct 2023 09:57:34 -0700 Subject: [PATCH 68/89] Update virtual provisioning counters on instance stop/start (#4277) Only charge virtual provisioning collections for instances when those instances are running. Charges are taken in the instance start saga and dropped when a sled agent tries to transition an instance to a stopped state. Unlike sled resource charges, provisioning charges are tied to instance states, not to VMM lifetimes. This ensures that a user is not charged twice for an instance (e.g. for quota management purposes) while the instance is migrating. See RFD 427 for more discussion. Also fix a small idempotency issue in the cleanup path for VMM resources. Tests: updated integration tests; manually checked virtual provisioning table values in a dev cluster & checked the values on the utilization graphs. Fixes #4257. --- nexus/db-model/src/schema.rs | 5 + .../virtual_provisioning_collection.rs | 13 +- .../virtual_provisioning_collection_update.rs | 29 ++++ nexus/src/app/instance.rs | 50 +++++- nexus/src/app/sagas/instance_create.rs | 56 ------- nexus/src/app/sagas/instance_delete.rs | 29 ---- nexus/src/app/sagas/instance_start.rs | 66 ++++++++ nexus/tests/integration_tests/instances.rs | 143 ++++++++++++++++-- 8 files changed, 285 insertions(+), 106 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 61a05754c6..9189b6db7b 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -812,6 +812,11 @@ table! { } } +allow_tables_to_appear_in_same_query! { + virtual_provisioning_resource, + instance +} + table! { zpool (id) { id -> Uuid, diff --git a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs index 18ff58735e..83856e10c7 100644 --- a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs +++ b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs @@ -272,7 +272,11 @@ impl DataStore { Ok(provisions) } - /// Transitively updates all CPU/RAM provisions from project -> fleet. + /// Transitively removes the CPU and memory charges for an instance from the + /// instance's project, silo, and fleet, provided that the instance's state + /// generation is less than `max_instance_gen`. This allows a caller who is + /// about to apply generation G to an instance to avoid deleting resources + /// if its update was superseded. pub async fn virtual_provisioning_collection_delete_instance( &self, opctx: &OpContext, @@ -280,10 +284,15 @@ impl DataStore { project_id: Uuid, cpus_diff: i64, ram_diff: ByteCount, + max_instance_gen: i64, ) -> Result, Error> { let provisions = VirtualProvisioningCollectionUpdate::new_delete_instance( - id, cpus_diff, ram_diff, project_id, + id, + max_instance_gen, + cpus_diff, + ram_diff, + project_id, ) .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await diff --git a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs index b7271f3f49..0a383eb6f1 100644 --- a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs +++ b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs @@ -368,10 +368,12 @@ impl VirtualProvisioningCollectionUpdate { pub fn new_delete_instance( id: uuid::Uuid, + max_instance_gen: i64, cpus_diff: i64, ram_diff: ByteCount, project_id: uuid::Uuid, ) -> Self { + use crate::db::schema::instance::dsl as instance_dsl; use virtual_provisioning_collection::dsl as collection_dsl; use virtual_provisioning_resource::dsl as resource_dsl; @@ -379,9 +381,36 @@ impl VirtualProvisioningCollectionUpdate { // We should delete the record if it exists. DoUpdate::new_for_delete(id), // The query to actually delete the record. + // + // The filter condition here ensures that the provisioning record is + // only deleted if the corresponding instance has a generation + // number less than the supplied `max_instance_gen`. This allows a + // caller that is about to apply an instance update that will stop + // the instance and that bears generation G to avoid deleting + // resources if the instance generation was already advanced to or + // past G. + // + // If the relevant instance ID is not in the database, then some + // other operation must have ensured the instance was previously + // stopped (because that's the only way it could have been deleted), + // and that operation should have cleaned up the resources already, + // in which case there's nothing to do here. + // + // There is an additional "direct" filter on the target resource ID + // to avoid a full scan of the resource table. UnreferenceableSubquery( diesel::delete(resource_dsl::virtual_provisioning_resource) .filter(resource_dsl::id.eq(id)) + .filter( + resource_dsl::id.nullable().eq(instance_dsl::instance + .filter(instance_dsl::id.eq(id)) + .filter( + instance_dsl::state_generation + .lt(max_instance_gen), + ) + .select(instance_dsl::id) + .single_value()), + ) .returning(virtual_provisioning_resource::all_columns), ), // Within this project, silo, fleet... diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 1adcd8f9c0..17d033c5a0 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1290,6 +1290,14 @@ impl super::Nexus { "propolis_id" => %propolis_id, "vmm_state" => ?new_runtime_state.vmm_state); + // Grab the current state of the instance in the DB to reason about + // whether this update is stale or not. + let (.., authz_instance, db_instance) = + LookupPath::new(&opctx, &self.db_datastore) + .instance_id(*instance_id) + .fetch() + .await?; + // Update OPTE and Dendrite if the instance's active sled assignment // changed or a migration was retired. If these actions fail, sled agent // is expected to retry this update. @@ -1303,12 +1311,6 @@ impl super::Nexus { // // In the future, this should be replaced by a call to trigger a // networking state update RPW. - let (.., authz_instance, db_instance) = - LookupPath::new(&opctx, &self.db_datastore) - .instance_id(*instance_id) - .fetch() - .await?; - self.ensure_updated_instance_network_config( opctx, &authz_instance, @@ -1317,6 +1319,27 @@ impl super::Nexus { ) .await?; + // If the supplied instance state indicates that the instance no longer + // has an active VMM, attempt to delete the virtual provisioning record + // + // As with updating networking state, this must be done before + // committing the new runtime state to the database: once the DB is + // written, a new start saga can arrive and start the instance, which + // will try to create its own virtual provisioning charges, which will + // race with this operation. + if new_runtime_state.instance_state.propolis_id.is_none() { + self.db_datastore + .virtual_provisioning_collection_delete_instance( + opctx, + *instance_id, + db_instance.project_id, + i64::from(db_instance.ncpus.0 .0), + db_instance.memory, + (&new_runtime_state.instance_state.gen).into(), + ) + .await?; + } + // Write the new instance and VMM states back to CRDB. This needs to be // done before trying to clean up the VMM, since the datastore will only // allow a VMM to be marked as deleted if it is already in a terminal @@ -1337,7 +1360,20 @@ impl super::Nexus { // If the VMM is now in a terminal state, make sure its resources get // cleaned up. - if let Ok((_, true)) = result { + // + // For idempotency, only check to see if the update was successfully + // processed and ignore whether the VMM record was actually updated. + // This is required to handle the case where this routine is called + // once, writes the terminal VMM state, fails before all per-VMM + // resources are released, returns a retriable error, and is retried: + // the per-VMM resources still need to be cleaned up, but the DB update + // will return Ok(_, false) because the database was already updated. + // + // Unlike the pre-update cases, it is legal to do this cleanup *after* + // committing state to the database, because a terminated VMM cannot be + // reused (restarting or migrating its former instance will use new VMM + // IDs). + if result.is_ok() { let propolis_terminated = matches!( new_runtime_state.vmm_state.state, InstanceState::Destroyed | InstanceState::Failed diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index 5d55aaf0fe..153e0323e7 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -13,7 +13,6 @@ use crate::external_api::params; use nexus_db_model::NetworkInterfaceKind; use nexus_db_queries::db::identity::Resource; use nexus_db_queries::db::lookup::LookupPath; -use nexus_db_queries::db::model::ByteCount as DbByteCount; use nexus_db_queries::db::queries::network_interface::InsertError as InsertNicError; use nexus_db_queries::{authn, authz, db}; use nexus_defaults::DEFAULT_PRIMARY_NIC_NAME; @@ -75,10 +74,6 @@ struct DiskAttachParams { declare_saga_actions! { instance_create; - VIRTUAL_RESOURCES_ACCOUNT -> "no_result" { - + sic_account_virtual_resources - - sic_account_virtual_resources_undo - } CREATE_INSTANCE_RECORD -> "instance_record" { + sic_create_instance_record - sic_delete_instance_record @@ -131,7 +126,6 @@ impl NexusSaga for SagaInstanceCreate { })?, )); - builder.append(virtual_resources_account_action()); builder.append(create_instance_record_action()); // Helper function for appending subsagas to our parent saga. @@ -728,56 +722,6 @@ async fn ensure_instance_disk_attach_state( Ok(()) } -async fn sic_account_virtual_resources( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let instance_id = sagactx.lookup::("instance_id")?; - - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - osagactx - .datastore() - .virtual_provisioning_collection_insert_instance( - &opctx, - instance_id, - params.project_id, - i64::from(params.create_params.ncpus.0), - DbByteCount(params.create_params.memory), - ) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - -async fn sic_account_virtual_resources_undo( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let instance_id = sagactx.lookup::("instance_id")?; - - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - osagactx - .datastore() - .virtual_provisioning_collection_delete_instance( - &opctx, - instance_id, - params.project_id, - i64::from(params.create_params.ncpus.0), - DbByteCount(params.create_params.memory), - ) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - async fn sic_create_instance_record( sagactx: NexusActionContext, ) -> Result { diff --git a/nexus/src/app/sagas/instance_delete.rs b/nexus/src/app/sagas/instance_delete.rs index 7da497136e..1605465c74 100644 --- a/nexus/src/app/sagas/instance_delete.rs +++ b/nexus/src/app/sagas/instance_delete.rs @@ -9,7 +9,6 @@ use super::NexusActionContext; use super::NexusSaga; use crate::app::sagas::declare_saga_actions; use nexus_db_queries::{authn, authz, db}; -use nexus_types::identity::Resource; use omicron_common::api::external::{Error, ResourceType}; use omicron_common::api::internal::shared::SwitchLocation; use serde::Deserialize; @@ -40,9 +39,6 @@ declare_saga_actions! { DEALLOCATE_EXTERNAL_IP -> "no_result3" { + sid_deallocate_external_ip } - VIRTUAL_RESOURCES_ACCOUNT -> "no_result4" { - + sid_account_virtual_resources - } } // instance delete saga: definition @@ -64,7 +60,6 @@ impl NexusSaga for SagaInstanceDelete { builder.append(instance_delete_record_action()); builder.append(delete_network_interfaces_action()); builder.append(deallocate_external_ip_action()); - builder.append(virtual_resources_account_action()); Ok(builder.build()?) } } @@ -135,30 +130,6 @@ async fn sid_deallocate_external_ip( Ok(()) } -async fn sid_account_virtual_resources( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - - osagactx - .datastore() - .virtual_provisioning_collection_delete_instance( - &opctx, - params.instance.id(), - params.instance.project_id, - i64::from(params.instance.ncpus.0 .0), - params.instance.memory, - ) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - #[cfg(test)] mod test { use crate::{ diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index 068d2e7005..76773d6369 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -52,6 +52,11 @@ declare_saga_actions! { - sis_move_to_starting_undo } + ADD_VIRTUAL_RESOURCES -> "virtual_resources" { + + sis_account_virtual_resources + - sis_account_virtual_resources_undo + } + // TODO(#3879) This can be replaced with an action that triggers the NAT RPW // once such an RPW is available. DPD_ENSURE -> "dpd_ensure" { @@ -98,6 +103,7 @@ impl NexusSaga for SagaInstanceStart { builder.append(alloc_propolis_ip_action()); builder.append(create_vmm_record_action()); builder.append(mark_as_starting_action()); + builder.append(add_virtual_resources_action()); builder.append(dpd_ensure_action()); builder.append(v2p_ensure_action()); builder.append(ensure_registered_action()); @@ -305,6 +311,66 @@ async fn sis_move_to_starting_undo( Ok(()) } +async fn sis_account_virtual_resources( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let instance_id = params.db_instance.id(); + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + osagactx + .datastore() + .virtual_provisioning_collection_insert_instance( + &opctx, + instance_id, + params.db_instance.project_id, + i64::from(params.db_instance.ncpus.0 .0), + nexus_db_model::ByteCount(*params.db_instance.memory), + ) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + +async fn sis_account_virtual_resources_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let instance_id = params.db_instance.id(); + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let started_record = + sagactx.lookup::("started_record")?; + + osagactx + .datastore() + .virtual_provisioning_collection_delete_instance( + &opctx, + instance_id, + params.db_instance.project_id, + i64::from(params.db_instance.ncpus.0 .0), + nexus_db_model::ByteCount(*params.db_instance.memory), + // Use the next instance generation number as the generation limit + // to ensure the provisioning counters are released. (The "mark as + // starting" undo step will "publish" this new state generation when + // it moves the instance back to Stopped.) + (&started_record.runtime().gen.next()).into(), + ) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + async fn sis_dpd_ensure( sagactx: NexusActionContext, ) -> Result<(), ActionError> { diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 9208e21652..ea633be9dc 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -942,7 +942,7 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { assert_metrics(cptestctx, project_id, 0, 0, 0).await; - // Create an instance. + // Create and start an instance. let instance_name = "just-rainsticks"; create_instance(client, PROJECT_NAME, instance_name).await; let virtual_provisioning_collection = datastore @@ -955,27 +955,22 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { ByteCount::from_gibibytes_u32(1), ); - // Stop the instance + // Stop the instance. This should cause the relevant resources to be + // deprovisioned. let instance = instance_post(&client, instance_name, InstanceOp::Stop).await; instance_simulate(nexus, &instance.identity.id).await; let instance = instance_get(&client, &get_instance_url(&instance_name)).await; assert_eq!(instance.runtime.run_state, InstanceState::Stopped); - // NOTE: I think it's arguably "more correct" to identify that the - // number of CPUs being used by guests at this point is actually "0", - // not "4", because the instance is stopped (same re: RAM usage). - // - // However, for implementation reasons, this is complicated (we have a - // tendency to update the runtime without checking the prior state, which - // makes edge-triggered behavior trickier to notice). + let virtual_provisioning_collection = datastore .virtual_provisioning_collection_get(&opctx, project_id) .await .unwrap(); - let expected_cpus = 4; + let expected_cpus = 0; let expected_ram = - i64::try_from(ByteCount::from_gibibytes_u32(1).to_bytes()).unwrap(); + i64::try_from(ByteCount::from_gibibytes_u32(0).to_bytes()).unwrap(); assert_eq!(virtual_provisioning_collection.cpus_provisioned, expected_cpus); assert_eq!( i64::from(virtual_provisioning_collection.ram_provisioned.0), @@ -983,7 +978,7 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { ); assert_metrics(cptestctx, project_id, 0, expected_cpus, expected_ram).await; - // Stop the instance + // Delete the instance. NexusRequest::object_delete(client, &get_instance_url(&instance_name)) .authn_as(AuthnMode::PrivilegedUser) .execute() @@ -999,6 +994,130 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { assert_metrics(cptestctx, project_id, 0, 0, 0).await; } +#[nexus_test] +async fn test_instance_metrics_with_migration( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + let instance_name = "bird-ecology"; + + // Create a second sled to migrate to/from. + let default_sled_id: Uuid = + nexus_test_utils::SLED_AGENT_UUID.parse().unwrap(); + let update_dir = Utf8Path::new("/should/be/unused"); + let other_sled_id = Uuid::new_v4(); + let _other_sa = nexus_test_utils::start_sled_agent( + cptestctx.logctx.log.new(o!("sled_id" => other_sled_id.to_string())), + cptestctx.server.get_http_server_internal_address().await, + other_sled_id, + &update_dir, + sim::SimMode::Explicit, + ) + .await + .unwrap(); + + let project_id = create_org_and_project(&client).await; + let instance_url = get_instance_url(instance_name); + + // Explicitly create an instance with no disks. Simulated sled agent assumes + // that disks are co-located with their instances. + let instance = nexus_test_utils::resource_helpers::create_instance_with( + client, + PROJECT_NAME, + instance_name, + ¶ms::InstanceNetworkInterfaceAttachment::Default, + Vec::::new(), + Vec::::new(), + ) + .await; + let instance_id = instance.identity.id; + + // Poke the instance into an active state. + instance_simulate(nexus, &instance_id).await; + let instance_next = instance_get(&client, &instance_url).await; + assert_eq!(instance_next.runtime.run_state, InstanceState::Running); + + // The instance should be provisioned while it's in the running state. + let nexus = &apictx.nexus; + let datastore = nexus.datastore(); + let check_provisioning_state = |cpus: i64, mem_gib: u32| async move { + let opctx = OpContext::for_tests( + cptestctx.logctx.log.new(o!()), + datastore.clone(), + ); + let virtual_provisioning_collection = datastore + .virtual_provisioning_collection_get(&opctx, project_id) + .await + .unwrap(); + assert_eq!( + virtual_provisioning_collection.cpus_provisioned, + cpus.clone() + ); + assert_eq!( + virtual_provisioning_collection.ram_provisioned.0, + ByteCount::from_gibibytes_u32(mem_gib) + ); + }; + + check_provisioning_state(4, 1).await; + + // Request migration to the other sled. This reserves resources on the + // target sled, but shouldn't change the virtual provisioning counters. + let original_sled = nexus + .instance_sled_id(&instance_id) + .await + .unwrap() + .expect("running instance should have a sled"); + + let dst_sled_id = if original_sled == default_sled_id { + other_sled_id + } else { + default_sled_id + }; + + let migrate_url = + format!("/v1/instances/{}/migrate", &instance_id.to_string()); + let _ = NexusRequest::new( + RequestBuilder::new(client, Method::POST, &migrate_url) + .body(Some(¶ms::InstanceMigrate { dst_sled_id })) + .expect_status(Some(StatusCode::OK)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body::() + .unwrap(); + + check_provisioning_state(4, 1).await; + + // Complete migration on the target. Simulated migrations always succeed. + // After this the instance should be running and should continue to appear + // to be provisioned. + instance_simulate_on_sled(cptestctx, nexus, dst_sled_id, instance_id).await; + let instance = instance_get(&client, &instance_url).await; + assert_eq!(instance.runtime.run_state, InstanceState::Running); + + check_provisioning_state(4, 1).await; + + // Now stop the instance. This should retire the instance's active Propolis + // and cause the virtual provisioning charges to be released. Note that + // the source sled still has an active resource charge for the source + // instance (whose demise wasn't simulated here), but this is intentionally + // not reflected in the virtual provisioning counters (which reflect the + // logical states of instances ignoring migration). + let instance = + instance_post(&client, instance_name, InstanceOp::Stop).await; + instance_simulate(nexus, &instance.identity.id).await; + let instance = + instance_get(&client, &get_instance_url(&instance_name)).await; + assert_eq!(instance.runtime.run_state, InstanceState::Stopped); + + check_provisioning_state(0, 0).await; +} + #[nexus_test] async fn test_instances_create_stopped_start( cptestctx: &ControlPlaneTestContext, From 0cfc8706242ac6f8b14657d3af065ab3a57507ba Mon Sep 17 00:00:00 2001 From: Ryan Goodfellow Date: Fri, 20 Oct 2023 20:48:10 -0700 Subject: [PATCH 69/89] BGP support (#3986) --- .../buildomat/jobs/build-and-test-linux.sh | 4 +- .github/buildomat/jobs/clippy.sh | 1 + .github/buildomat/jobs/deploy.sh | 10 +- .github/buildomat/jobs/package.sh | 6 +- .gitignore | 2 + Cargo.lock | 26 + Cargo.toml | 3 + bootstore/src/schemes/v0/storage.rs | 4 +- clients/bootstrap-agent-client/src/lib.rs | 2 + clients/ddm-admin-client/build.rs | 17 +- clients/mg-admin-client/Cargo.toml | 26 + clients/mg-admin-client/build.rs | 102 ++ clients/mg-admin-client/src/lib.rs | 83 ++ clients/nexus-client/src/lib.rs | 2 + clients/sled-agent-client/src/lib.rs | 32 +- clients/wicketd-client/src/lib.rs | 8 +- common/src/address.rs | 9 + common/src/api/external/mod.rs | 85 +- common/src/api/internal/shared.rs | 110 +- common/src/nexus_config.rs | 20 +- dev-tools/omdb/tests/env.out | 18 +- dev-tools/omdb/tests/successes.out | 20 +- env.sh | 1 + illumos-utils/src/destructor.rs | 2 +- installinator/src/bootstrap.rs | 2 +- installinator/src/dispatch.rs | 2 +- internal-dns/src/config.rs | 34 +- internal-dns/src/names.rs | 13 +- nexus/Cargo.toml | 1 + nexus/db-macros/src/lookup.rs | 4 +- nexus/db-model/src/bgp.rs | 37 + nexus/db-model/src/bootstore.rs | 13 + nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/rack.rs | 3 + nexus/db-model/src/schema.rs | 21 +- nexus/db-model/src/service_kind.rs | 2 + nexus/db-model/src/switch_interface.rs | 9 +- nexus/db-model/src/switch_port.rs | 188 +++- nexus/db-queries/src/db/datastore/bgp.rs | 351 +++++++ .../db-queries/src/db/datastore/bootstore.rs | 37 + nexus/db-queries/src/db/datastore/mod.rs | 2 + nexus/db-queries/src/db/datastore/rack.rs | 25 + .../src/db/datastore/switch_port.rs | 294 +++--- nexus/src/app/bgp.rs | 162 +++ nexus/src/app/mod.rs | 67 +- nexus/src/app/rack.rs | 226 ++++- nexus/src/app/sagas/mod.rs | 1 - .../app/sagas/switch_port_settings_apply.rs | 760 +++++++++++++- .../app/sagas/switch_port_settings_clear.rs | 209 +++- .../app/sagas/switch_port_settings_update.rs | 5 - nexus/src/app/switch_port.rs | 110 +- nexus/src/external_api/http_entrypoints.rs | 203 +++- nexus/src/lib.rs | 12 +- nexus/test-utils/src/lib.rs | 62 ++ nexus/tests/integration_tests/address_lots.rs | 10 +- nexus/tests/integration_tests/endpoints.rs | 77 ++ .../tests/integration_tests/initialization.rs | 9 + nexus/tests/integration_tests/schema.rs | 2 + nexus/tests/integration_tests/switch_port.rs | 92 +- nexus/tests/output/nexus_tags.txt | 8 + nexus/types/src/external_api/params.rs | 111 ++- nexus/types/src/internal_api/params.rs | 2 + openapi/bootstrap-agent.json | 239 +++-- openapi/nexus-internal.json | 253 +++-- openapi/nexus.json | 938 +++++++++++++++++- openapi/sled-agent.json | 401 ++++++++ openapi/wicketd.json | 241 +++-- package-manifest.toml | 40 +- schema/crdb/8.0.0/up01.sql | 1 + schema/crdb/8.0.0/up02.sql | 1 + schema/crdb/8.0.0/up03.sql | 1 + schema/crdb/8.0.0/up04.sql | 5 + schema/crdb/8.0.0/up05.sql | 11 + schema/crdb/8.0.0/up06.sql | 1 + schema/crdb/8.0.0/up07.sql | 1 + schema/crdb/8.0.0/up08.sql | 1 + schema/crdb/8.0.0/up09.sql | 1 + schema/crdb/8.0.0/up10.sql | 1 + schema/crdb/8.0.0/up11.sql | 1 + schema/crdb/8.0.0/up12.sql | 1 + schema/crdb/8.0.0/up13.sql | 1 + schema/crdb/8.0.0/up14.sql | 4 + schema/crdb/dbinit.sql | 45 +- schema/rss-sled-plan.json | 243 +++-- sled-agent/Cargo.toml | 2 +- sled-agent/src/bootstrap/early_networking.rs | 290 +++++- sled-agent/src/bootstrap/maghemite.rs | 2 +- sled-agent/src/bootstrap/secret_retriever.rs | 6 +- sled-agent/src/bootstrap/server.rs | 2 +- sled-agent/src/http_entrypoints.rs | 82 +- sled-agent/src/params.rs | 21 +- sled-agent/src/rack_setup/plan/service.rs | 17 +- sled-agent/src/rack_setup/service.rs | 51 +- sled-agent/src/services.rs | 67 +- sled-agent/src/sim/http_entrypoints.rs | 57 ++ sled-agent/src/sim/server.rs | 2 +- sled-agent/src/sled_agent.rs | 27 +- .../gimlet-standalone/config-rss.toml | 21 +- smf/sled-agent/non-gimlet/config-rss.toml | 21 +- test-utils/src/dev/dendrite.rs | 2 +- test-utils/src/dev/maghemite.rs | 155 +++ test-utils/src/dev/mod.rs | 1 + tools/build-global-zone-packages.sh | 4 +- .../build-trampoline-global-zone-packages.sh | 4 +- tools/ci_download_maghemite_mgd | 168 ++++ tools/ci_download_maghemite_openapi | 13 +- tools/ci_download_softnpu_machinery | 2 +- tools/create_virtual_hardware.sh | 6 +- tools/delete-reservoir.sh | 6 + tools/dendrite_openapi_version | 2 +- tools/dendrite_stub_checksums | 6 +- tools/install_builder_prerequisites.sh | 4 + tools/install_runner_prerequisites.sh | 5 +- ..._version => maghemite_ddm_openapi_version} | 2 +- tools/maghemite_mg_openapi_version | 2 + tools/maghemite_mgd_checksums | 2 + tools/update_maghemite.sh | 33 +- update-engine/src/context.rs | 2 +- wicket/src/rack_setup/config_template.toml | 30 +- wicket/src/rack_setup/config_toml.rs | 186 +++- wicket/src/ui/main.rs | 2 +- wicket/src/ui/panes/rack_setup.rs | 63 +- wicket/src/ui/wrap.rs | 2 +- wicketd/Cargo.toml | 1 + wicketd/src/installinator_progress.rs | 2 +- wicketd/src/preflight_check/uplink.rs | 275 ++--- wicketd/src/rss_config.rs | 69 +- workspace-hack/Cargo.toml | 2 + 128 files changed, 6883 insertions(+), 1028 deletions(-) create mode 100644 clients/mg-admin-client/Cargo.toml create mode 100644 clients/mg-admin-client/build.rs create mode 100644 clients/mg-admin-client/src/lib.rs create mode 100644 nexus/db-model/src/bootstore.rs create mode 100644 nexus/db-queries/src/db/datastore/bgp.rs create mode 100644 nexus/db-queries/src/db/datastore/bootstore.rs create mode 100644 nexus/src/app/bgp.rs delete mode 100644 nexus/src/app/sagas/switch_port_settings_update.rs create mode 100644 schema/crdb/8.0.0/up01.sql create mode 100644 schema/crdb/8.0.0/up02.sql create mode 100644 schema/crdb/8.0.0/up03.sql create mode 100644 schema/crdb/8.0.0/up04.sql create mode 100644 schema/crdb/8.0.0/up05.sql create mode 100644 schema/crdb/8.0.0/up06.sql create mode 100644 schema/crdb/8.0.0/up07.sql create mode 100644 schema/crdb/8.0.0/up08.sql create mode 100644 schema/crdb/8.0.0/up09.sql create mode 100644 schema/crdb/8.0.0/up10.sql create mode 100644 schema/crdb/8.0.0/up11.sql create mode 100644 schema/crdb/8.0.0/up12.sql create mode 100644 schema/crdb/8.0.0/up13.sql create mode 100644 schema/crdb/8.0.0/up14.sql create mode 100644 test-utils/src/dev/maghemite.rs create mode 100755 tools/ci_download_maghemite_mgd create mode 100755 tools/delete-reservoir.sh rename tools/{maghemite_openapi_version => maghemite_ddm_openapi_version} (59%) create mode 100644 tools/maghemite_mg_openapi_version create mode 100644 tools/maghemite_mgd_checksums diff --git a/.github/buildomat/jobs/build-and-test-linux.sh b/.github/buildomat/jobs/build-and-test-linux.sh index f33d1a8cfa..715effd080 100755 --- a/.github/buildomat/jobs/build-and-test-linux.sh +++ b/.github/buildomat/jobs/build-and-test-linux.sh @@ -1,8 +1,8 @@ #!/bin/bash #: -#: name = "build-and-test (ubuntu-20.04)" +#: name = "build-and-test (ubuntu-22.04)" #: variety = "basic" -#: target = "ubuntu-20.04" +#: target = "ubuntu-22.04" #: rust_toolchain = "1.72.1" #: output_rules = [ #: "/var/tmp/omicron_tmp/*", diff --git a/.github/buildomat/jobs/clippy.sh b/.github/buildomat/jobs/clippy.sh index dba1021919..5fd31adb76 100755 --- a/.github/buildomat/jobs/clippy.sh +++ b/.github/buildomat/jobs/clippy.sh @@ -29,3 +29,4 @@ ptime -m bash ./tools/install_builder_prerequisites.sh -y banner clippy ptime -m cargo xtask clippy +ptime -m cargo doc diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index bdc1a9cce8..ff9b44fc40 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -232,11 +232,11 @@ infra_ip_first = \"$UPLINK_IP\" /^infra_ip_last/c\\ infra_ip_last = \"$UPLINK_IP\" } - /^\\[\\[rack_network_config.uplinks/,/^\$/ { - /^gateway_ip/c\\ -gateway_ip = \"$GATEWAY_IP\" - /^uplink_cidr/c\\ -uplink_cidr = \"$UPLINK_IP/32\" + /^\\[\\[rack_network_config.ports/,/^\$/ { + /^routes/c\\ +routes = \\[{nexthop = \"$GATEWAY_IP\", destination = \"0.0.0.0/0\"}\\] + /^addresses/c\\ +addresses = \\[\"$UPLINK_IP/32\"\\] } " pkg/config-rss.toml diff -u pkg/config-rss.toml{~,} || true diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 64c087524e..c1cb04124d 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -71,7 +71,7 @@ tarball_src_dir="$(pwd)/out/versioned" stamp_packages() { for package in "$@"; do # TODO: remove once https://github.com/oxidecomputer/omicron-package/pull/54 lands - if [[ $package == maghemite ]]; then + if [[ $package == mg-ddm-gz ]]; then echo "0.0.0" > VERSION tar rvf "out/$package.tar" VERSION rm VERSION @@ -90,7 +90,7 @@ ptime -m cargo run --locked --release --bin omicron-package -- \ -t host target create -i standard -m gimlet -s asic -r multi-sled ptime -m cargo run --locked --release --bin omicron-package -- \ -t host package -stamp_packages omicron-sled-agent maghemite propolis-server overlay +stamp_packages omicron-sled-agent mg-ddm-gz propolis-server overlay # Create global zone package @ /work/global-zone-packages.tar.gz ptime -m ./tools/build-global-zone-packages.sh "$tarball_src_dir" /work @@ -135,7 +135,7 @@ ptime -m cargo run --locked --release --bin omicron-package -- \ -t recovery target create -i trampoline ptime -m cargo run --locked --release --bin omicron-package -- \ -t recovery package -stamp_packages installinator maghemite +stamp_packages installinator mg-ddm-gz # Create trampoline global zone package @ /work/trampoline-global-zone-packages.tar.gz ptime -m ./tools/build-trampoline-global-zone-packages.sh "$tarball_src_dir" /work diff --git a/.gitignore b/.gitignore index 574e867c02..1d7177320f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ core *.vdev debug.out rusty-tags.vi +*.sw* +tags diff --git a/Cargo.lock b/Cargo.lock index b38cda6a45..06d5f2fb70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4161,6 +4161,29 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mg-admin-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "either", + "omicron-common 0.1.0", + "omicron-workspace-hack", + "omicron-zone-package", + "progenitor", + "progenitor-client", + "quote", + "reqwest", + "rustfmt-wrapper", + "serde", + "serde_json", + "sled-hardware", + "slog", + "thiserror", + "tokio", + "toml 0.7.8", +] + [[package]] name = "mime" version = "0.3.17" @@ -5078,6 +5101,7 @@ dependencies = [ "itertools 0.11.0", "lazy_static", "macaddr", + "mg-admin-client", "mime_guess", "newtype_derive", "nexus-db-model", @@ -5460,6 +5484,7 @@ dependencies = [ "schemars", "semver 1.0.18", "serde", + "serde_json", "sha2", "signature 2.1.0", "similar", @@ -10116,6 +10141,7 @@ dependencies = [ "installinator-artifactd", "installinator-common", "internal-dns 0.1.0", + "ipnetwork", "itertools 0.11.0", "omicron-certificates", "omicron-common 0.1.0", diff --git a/Cargo.toml b/Cargo.toml index b213f3adff..e9eea3c4ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "clients/dpd-client", "clients/gateway-client", "clients/installinator-artifact-client", + "clients/mg-admin-client", "clients/nexus-client", "clients/oxide-client", "clients/oximeter-client", @@ -82,6 +83,7 @@ default-members = [ "clients/oximeter-client", "clients/sled-agent-client", "clients/wicketd-client", + "clients/mg-admin-client", "common", "dev-tools/crdb-seed", "dev-tools/omdb", @@ -227,6 +229,7 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } mime_guess = "2.0.4" mockall = "0.11" newtype_derive = "0.1.6" +mg-admin-client = { path = "clients/mg-admin-client" } nexus-client = { path = "clients/nexus-client" } nexus-db-model = { path = "nexus/db-model" } nexus-db-queries = { path = "nexus/db-queries" } diff --git a/bootstore/src/schemes/v0/storage.rs b/bootstore/src/schemes/v0/storage.rs index ee31d24f05..327acc6058 100644 --- a/bootstore/src/schemes/v0/storage.rs +++ b/bootstore/src/schemes/v0/storage.rs @@ -5,9 +5,9 @@ //! Storage for the v0 bootstore scheme //! //! We write two pieces of data to M.2 devices in production via -//! [`omicron_common::Ledger`]: +//! [`omicron_common::ledger::Ledger`]: //! -//! 1. [`super::Fsm::State`] for bootstore state itself +//! 1. [`super::State`] for bootstore state itself //! 2. A network config blob required for pre-rack-unlock configuration //! diff --git a/clients/bootstrap-agent-client/src/lib.rs b/clients/bootstrap-agent-client/src/lib.rs index 3f8b20e1f5..19ecb599f3 100644 --- a/clients/bootstrap-agent-client/src/lib.rs +++ b/clients/bootstrap-agent-client/src/lib.rs @@ -20,6 +20,8 @@ progenitor::generate_api!( derives = [schemars::JsonSchema], replace = { Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, } ); diff --git a/clients/ddm-admin-client/build.rs b/clients/ddm-admin-client/build.rs index e3c1345eda..da74ee9962 100644 --- a/clients/ddm-admin-client/build.rs +++ b/clients/ddm-admin-client/build.rs @@ -21,20 +21,21 @@ fn main() -> Result<()> { println!("cargo:rerun-if-changed=../../package-manifest.toml"); let config: Config = toml::from_str(&manifest) - .context("failed to parse ../../package-manifest.toml")?; - let maghemite = config + .context("failed to parse ../package-manifest.toml")?; + + let ddm = config .packages - .get("maghemite") - .context("missing maghemite package in ../../package-manifest.toml")?; + .get("mg-ddm-gz") + .context("missing mg-ddm-gz package in ../package-manifest.toml")?; - let local_path = match &maghemite.source { + let local_path = match &ddm.source { PackageSource::Prebuilt { commit, .. } => { // Report a relatively verbose error if we haven't downloaded the requisite // openapi spec. let local_path = format!("../../out/downloads/ddm-admin-{commit}.json"); if !Path::new(&local_path).exists() { - bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); + bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_ddm_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); } println!("cargo:rerun-if-changed={local_path}"); local_path @@ -51,7 +52,9 @@ fn main() -> Result<()> { } _ => { - bail!("maghemite external package must have type `prebuilt` or `manual`") + bail!( + "mg-ddm external package must have type `prebuilt` or `manual`" + ) } }; diff --git a/clients/mg-admin-client/Cargo.toml b/clients/mg-admin-client/Cargo.toml new file mode 100644 index 0000000000..c444fee32f --- /dev/null +++ b/clients/mg-admin-client/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "mg-admin-client" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[dependencies] +either.workspace = true +progenitor-client.workspace = true +reqwest = { workspace = true, features = ["json", "stream", "rustls-tls"] } +serde.workspace = true +slog.workspace = true +thiserror.workspace = true +tokio.workspace = true +omicron-common.workspace = true +sled-hardware.workspace = true +omicron-workspace-hack.workspace = true + +[build-dependencies] +anyhow.workspace = true +omicron-zone-package.workspace = true +progenitor.workspace = true +quote.workspace = true +rustfmt-wrapper.workspace = true +serde_json.workspace = true +toml.workspace = true diff --git a/clients/mg-admin-client/build.rs b/clients/mg-admin-client/build.rs new file mode 100644 index 0000000000..dcc7ae61cb --- /dev/null +++ b/clients/mg-admin-client/build.rs @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2022 Oxide Computer Company + +use anyhow::bail; +use anyhow::Context; +use anyhow::Result; +use omicron_zone_package::config::Config; +use omicron_zone_package::package::PackageSource; +use quote::quote; +use std::env; +use std::fs; +use std::path::Path; + +fn main() -> Result<()> { + // Find the current maghemite repo commit from our package manifest. + let manifest = fs::read_to_string("../../package-manifest.toml") + .context("failed to read ../../package-manifest.toml")?; + println!("cargo:rerun-if-changed=../../package-manifest.toml"); + + let config: Config = toml::from_str(&manifest) + .context("failed to parse ../../package-manifest.toml")?; + let mg = config + .packages + .get("mgd") + .context("missing mgd package in ../../package-manifest.toml")?; + + let local_path = match &mg.source { + PackageSource::Prebuilt { commit, .. } => { + // Report a relatively verbose error if we haven't downloaded the requisite + // openapi spec. + let local_path = + format!("../../out/downloads/mg-admin-{commit}.json"); + if !Path::new(&local_path).exists() { + bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_mg_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); + } + println!("cargo:rerun-if-changed={local_path}"); + local_path + } + + PackageSource::Manual => { + let local_path = + "../../out/downloads/mg-admin-manual.json".to_string(); + if !Path::new(&local_path).exists() { + bail!("{local_path} doesn't exist, please copy manually built mg-admin.json there!"); + } + println!("cargo:rerun-if-changed={local_path}"); + local_path + } + + _ => { + bail!("mgd external package must have type `prebuilt` or `manual`") + } + }; + + let spec = { + let bytes = fs::read(&local_path) + .with_context(|| format!("failed to read {local_path}"))?; + serde_json::from_slice(&bytes).with_context(|| { + format!("failed to parse {local_path} as openapi spec") + })? + }; + + let code = progenitor::Generator::new( + progenitor::GenerationSettings::new() + .with_inner_type(quote!(slog::Logger)) + .with_pre_hook(quote! { + |log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + } + }) + .with_post_hook(quote! { + |log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + } + }), + ) + .generate_tokens(&spec) + .with_context(|| { + format!("failed to generate progenitor client from {local_path}") + })?; + + let content = rustfmt_wrapper::rustfmt(code).with_context(|| { + format!("rustfmt failed on progenitor code from {local_path}") + })?; + + let out_file = + Path::new(&env::var("OUT_DIR").expect("OUT_DIR env var not set")) + .join("mg-admin-client.rs"); + + fs::write(&out_file, content).with_context(|| { + format!("failed to write client to {}", out_file.display()) + })?; + + Ok(()) +} diff --git a/clients/mg-admin-client/src/lib.rs b/clients/mg-admin-client/src/lib.rs new file mode 100644 index 0000000000..bb1d925c73 --- /dev/null +++ b/clients/mg-admin-client/src/lib.rs @@ -0,0 +1,83 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +#![allow(clippy::redundant_closure_call)] +#![allow(clippy::needless_lifetimes)] +#![allow(clippy::match_single_binding)] +#![allow(clippy::clone_on_copy)] +#![allow(rustdoc::broken_intra_doc_links)] +#![allow(rustdoc::invalid_html_tags)] + +#[allow(dead_code)] +mod inner { + include!(concat!(env!("OUT_DIR"), "/mg-admin-client.rs")); +} + +pub use inner::types; +pub use inner::Error; + +use inner::Client as InnerClient; +use omicron_common::api::external::BgpPeerState; +use slog::Logger; +use std::net::Ipv6Addr; +use std::net::SocketAddr; +use thiserror::Error; + +// TODO-cleanup Is it okay to hardcode this port number here? +const MGD_PORT: u16 = 4676; + +#[derive(Debug, Error)] +pub enum MgError { + #[error("Failed to construct an HTTP client: {0}")] + HttpClient(#[from] reqwest::Error), + + #[error("Failed making HTTP request to mgd: {0}")] + MgApi(#[from] Error), +} + +impl From for BgpPeerState { + fn from(s: inner::types::FsmStateKind) -> BgpPeerState { + use inner::types::FsmStateKind; + match s { + FsmStateKind::Idle => BgpPeerState::Idle, + FsmStateKind::Connect => BgpPeerState::Connect, + FsmStateKind::Active => BgpPeerState::Active, + FsmStateKind::OpenSent => BgpPeerState::OpenSent, + FsmStateKind::OpenConfirm => BgpPeerState::OpenConfirm, + FsmStateKind::SessionSetup => BgpPeerState::SessionSetup, + FsmStateKind::Established => BgpPeerState::Established, + } + } +} + +#[derive(Debug, Clone)] +pub struct Client { + pub inner: InnerClient, + pub log: Logger, +} + +impl Client { + /// Creates a new [`Client`] that points to localhost + pub fn localhost(log: &Logger) -> Result { + Self::new(log, SocketAddr::new(Ipv6Addr::LOCALHOST.into(), MGD_PORT)) + } + + pub fn new(log: &Logger, mgd_addr: SocketAddr) -> Result { + let dur = std::time::Duration::from_secs(60); + let log = log.new(slog::o!("MgAdminClient" => mgd_addr)); + + let inner = reqwest::ClientBuilder::new() + .connect_timeout(dur) + .timeout(dur) + .build()?; + let inner = InnerClient::new_with_client( + &format!("http://{mgd_addr}"), + inner, + log.clone(), + ); + Ok(Self { inner, log }) + } +} diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs index 33a68cb3ce..23ceb114fc 100644 --- a/clients/nexus-client/src/lib.rs +++ b/clients/nexus-client/src/lib.rs @@ -23,6 +23,8 @@ progenitor::generate_api!( }), replace = { Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, MacAddr = omicron_common::api::external::MacAddr, Name = omicron_common::api::external::Name, NewPasswordHash = omicron_passwords::NewPasswordHash, diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 3daac7dd60..0df21d894e 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -5,11 +5,33 @@ //! Interface for making API requests to a Sled Agent use async_trait::async_trait; -use omicron_common::generate_logging_api; use std::convert::TryFrom; use uuid::Uuid; -generate_logging_api!("../../openapi/sled-agent.json"); +progenitor::generate_api!( + spec = "../../openapi/sled-agent.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + //TODO trade the manual transformations later in this file for the + // replace directives below? + replace = { + //Ipv4Network = ipnetwork::Ipv4Network, + SwitchLocation = omicron_common::api::external::SwitchLocation, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, + PortFec = omicron_common::api::internal::shared::PortFec, + PortSpeed = omicron_common::api::internal::shared::PortSpeed, + } +); impl omicron_common::api::external::ClientError for types::Error { fn message(&self) -> String { @@ -269,6 +291,12 @@ impl From for types::Ipv4Net { } } +impl From for types::Ipv4Network { + fn from(n: ipnetwork::Ipv4Network) -> Self { + Self::try_from(n.to_string()).unwrap_or_else(|e| panic!("{}: {}", n, e)) + } +} + impl From for types::Ipv6Net { fn from(n: ipnetwork::Ipv6Network) -> Self { Self::try_from(n.to_string()).unwrap_or_else(|e| panic!("{}: {}", n, e)) diff --git a/clients/wicketd-client/src/lib.rs b/clients/wicketd-client/src/lib.rs index ff45232520..982ec13780 100644 --- a/clients/wicketd-client/src/lib.rs +++ b/clients/wicketd-client/src/lib.rs @@ -42,8 +42,12 @@ progenitor::generate_api!( RackInitId = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, RackResetId = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, RackOperationStatus = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, - RackNetworkConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + RackNetworkConfigV1 = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, UplinkConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + PortConfigV1 = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + BgpPeerConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + BgpConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + RouteConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfigInsensitive = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfigSensitive = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, @@ -52,6 +56,8 @@ progenitor::generate_api!( replace = { Duration = std::time::Duration, Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, PutRssUserConfigInsensitive = wicket_common::rack_setup::PutRssUserConfigInsensitive, EventReportForWicketdEngineSpec = wicket_common::update_events::EventReport, StepEventForWicketdEngineSpec = wicket_common::update_events::StepEvent, diff --git a/common/src/address.rs b/common/src/address.rs index baa344ef22..992e8f0406 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -39,6 +39,7 @@ pub const CLICKHOUSE_PORT: u16 = 8123; pub const CLICKHOUSE_KEEPER_PORT: u16 = 9181; pub const OXIMETER_PORT: u16 = 12223; pub const DENDRITE_PORT: u16 = 12224; +pub const MGD_PORT: u16 = 4676; pub const DDMD_PORT: u16 = 8000; pub const MGS_PORT: u16 = 12225; pub const WICKETD_PORT: u16 = 12226; @@ -172,6 +173,14 @@ impl Ipv6Subnet { } } +impl From for Ipv6Subnet { + fn from(net: Ipv6Network) -> Self { + // Ensure the address is set to within-prefix only components. + let net = Ipv6Network::new(net.network(), N).unwrap(); + Self { net: Ipv6Net(net) } + } +} + // We need a custom Deserialize to ensure that the subnet is what we expect. impl<'de, const N: u8> Deserialize<'de> for Ipv6Subnet { fn deserialize(deserializer: D) -> Result diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 53512408af..fcea57220d 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -12,6 +12,7 @@ pub mod http_pagination; use dropshot::HttpError; pub use error::*; +pub use crate::api::internal::shared::SwitchLocation; use anyhow::anyhow; use anyhow::Context; use api_identity::ObjectIdentity; @@ -98,6 +99,13 @@ pub struct DataPageParams<'a, NameType> { } impl<'a, NameType> DataPageParams<'a, NameType> { + pub fn max_page() -> Self { + Self { + marker: None, + direction: dropshot::PaginationOrder::Ascending, + limit: NonZeroU32::new(u32::MAX).unwrap(), + } + } /// Maps the marker type to a new type. /// /// Equivalent to [std::option::Option::map], because that's what it calls. @@ -400,7 +408,7 @@ impl SemverVersion { /// This is the official ECMAScript-compatible validation regex for /// semver: - /// https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + /// const VALIDATION_REGEX: &str = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"; } @@ -690,6 +698,8 @@ pub enum ResourceType { AddressLot, AddressLotBlock, BackgroundTask, + BgpConfig, + BgpAnnounceSet, Fleet, Silo, SiloUser, @@ -2459,9 +2469,6 @@ pub struct SwitchPortBgpPeerConfig { /// The port settings object this BGP configuration belongs to. pub port_settings_id: Uuid, - /// The id for the set of prefixes announced in this peer configuration. - pub bgp_announce_set_id: Uuid, - /// The id of the global BGP configuration referenced by this peer /// configuration. pub bgp_config_id: Uuid, @@ -2476,7 +2483,9 @@ pub struct SwitchPortBgpPeerConfig { } /// A base BGP configuration. -#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +#[derive( + ObjectIdentity, Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, +)] pub struct BgpConfig { #[serde(flatten)] pub identity: IdentityMetadata, @@ -2528,6 +2537,72 @@ pub struct SwitchPortAddressConfig { pub interface_name: String, } +/// The current state of a BGP peer. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum BgpPeerState { + /// Initial state. Refuse all incomming BGP connections. No resources + /// allocated to peer. + Idle, + + /// Waiting for the TCP connection to be completed. + Connect, + + /// Trying to acquire peer by listening for and accepting a TCP connection. + Active, + + /// Waiting for open message from peer. + OpenSent, + + /// Waiting for keepaliave or notification from peer. + OpenConfirm, + + /// Synchronizing with peer. + SessionSetup, + + /// Session established. Able to exchange update, notification and keepliave + /// messages with peers. + Established, +} + +/// The current status of a BGP peer. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct BgpPeerStatus { + /// IP address of the peer. + pub addr: IpAddr, + + /// Local autonomous system number. + pub local_asn: u32, + + /// Remote autonomous system number. + pub remote_asn: u32, + + /// State of the peer. + pub state: BgpPeerState, + + /// Time of last state change. + pub state_duration_millis: u64, + + /// Switch with the peer session. + pub switch: SwitchLocation, +} + +/// A route imported from a BGP peer. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct BgpImportedRouteIpv4 { + /// The destination network prefix. + pub prefix: Ipv4Net, + + /// The nexthop the prefix is reachable through. + pub nexthop: Ipv4Addr, + + /// BGP identifier of the originating router. + pub id: u32, + + /// Switch the route is imported into. + pub switch: SwitchLocation, +} + #[cfg(test)] mod test { use serde::Deserialize; diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 9e3f3ec1f6..784da8fcc6 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -5,7 +5,7 @@ //! Types shared between Nexus and Sled Agent. use crate::api::external::{self, Name}; -use ipnetwork::Ipv4Network; +use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{ @@ -68,18 +68,88 @@ pub struct SourceNatConfig { pub last_port: u16, } +// We alias [`RackNetworkConfig`] to the current version of the protocol, so +// that we can convert between versions as necessary. +pub type RackNetworkConfig = RackNetworkConfigV1; + /// Initial network configuration #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] -pub struct RackNetworkConfig { +pub struct RackNetworkConfigV1 { + pub rack_subnet: Ipv6Network, // TODO: #3591 Consider making infra-ip ranges implicit for uplinks /// First ip address to be used for configuring network infrastructure pub infra_ip_first: Ipv4Addr, /// Last ip address to be used for configuring network infrastructure pub infra_ip_last: Ipv4Addr, /// Uplinks for connecting the rack to external networks - pub uplinks: Vec, + pub ports: Vec, + /// BGP configurations for connecting the rack to external networks + pub bgp: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct BgpConfig { + /// The autonomous system number for the BGP configuration. + pub asn: u32, + /// The set of prefixes for the BGP router to originate. + pub originate: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct BgpPeerConfig { + /// The autonomous sysetm number of the router the peer belongs to. + pub asn: u32, + /// Switch port the peer is reachable on. + pub port: String, + /// Address of the peer. + pub addr: Ipv4Addr, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct RouteConfig { + /// The destination of the route. + pub destination: IpNetwork, + /// The nexthop/gateway address. + pub nexthop: IpAddr, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct PortConfigV1 { + /// The set of routes associated with this port. + pub routes: Vec, + /// This port's addresses. + pub addresses: Vec, + /// Switch the port belongs to. + pub switch: SwitchLocation, + /// Nmae of the port this config applies to. + pub port: String, + /// Port speed. + pub uplink_port_speed: PortSpeed, + /// Port forward error correction type. + pub uplink_port_fec: PortFec, + /// BGP peers on this port + pub bgp_peers: Vec, } +impl From for PortConfigV1 { + fn from(value: UplinkConfig) -> Self { + PortConfigV1 { + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: value.gateway_ip.into(), + }], + addresses: vec![value.uplink_cidr.into()], + switch: value.switch, + port: value.uplink_port, + uplink_port_speed: value.uplink_port_speed, + uplink_port_fec: value.uplink_port_fec, + bgp_peers: vec![], + } + } +} + +/// Deprecated, use PortConfigV1 instead. Cannot actually deprecate due to +/// #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct UplinkConfig { /// Gateway address @@ -99,9 +169,41 @@ pub struct UplinkConfig { pub uplink_vid: Option, } +/// A set of switch uplinks. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct SwitchPorts { + pub uplinks: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct HostPortConfig { + /// Switchport to use for external connectivity + pub port: String, + + /// IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport + /// (must be in infra_ip pool) + pub addrs: Vec, +} + +impl From for HostPortConfig { + fn from(x: PortConfigV1) -> Self { + Self { port: x.port, addrs: x.addresses } + } +} + /// Identifies switch physical location #[derive( - Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema, Hash, Eq, + Clone, + Copy, + Debug, + Deserialize, + Serialize, + PartialEq, + JsonSchema, + Hash, + Eq, + PartialOrd, + Ord, )] #[serde(rename_all = "snake_case")] pub enum SwitchLocation { diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index 6b0960643e..da50356d2e 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -241,6 +241,12 @@ pub struct DpdConfig { pub address: SocketAddr, } +/// Configuration for the `Dendrite` dataplane daemon. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct MgdConfig { + pub address: SocketAddr, +} + // A deserializable type that does no validation on the tunable parameters. #[derive(Clone, Debug, Deserialize, PartialEq)] struct UnvalidatedTunables { @@ -388,6 +394,9 @@ pub struct PackageConfig { /// `Dendrite` dataplane daemon configuration #[serde(default)] pub dendrite: HashMap, + /// Maghemite mgd daemon configuration + #[serde(default)] + pub mgd: HashMap, /// Background task configuration pub background_tasks: BackgroundTaskConfig, /// Default Crucible region allocation strategy @@ -469,7 +478,7 @@ mod test { use crate::nexus_config::{ BackgroundTaskConfig, ConfigDropshotWithTls, Database, DeploymentConfig, DnsTasksConfig, DpdConfig, ExternalEndpointsConfig, - InternalDns, LoadErrorKind, + InternalDns, LoadErrorKind, MgdConfig, }; use dropshot::ConfigDropshot; use dropshot::ConfigLogging; @@ -605,6 +614,8 @@ mod test { type = "from_dns" [dendrite.switch0] address = "[::1]:12224" + [mgd.switch0] + address = "[::1]:4676" [background_tasks] dns_internal.period_secs_config = 1 dns_internal.period_secs_servers = 2 @@ -686,6 +697,13 @@ mod test { .unwrap(), } )]), + mgd: HashMap::from([( + SwitchLocation::Switch0, + MgdConfig { + address: SocketAddr::from_str("[::1]:4676") + .unwrap(), + } + )]), background_tasks: BackgroundTaskConfig { dns_internal: DnsTasksConfig { period_secs_config: Duration::from_secs(1), diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 8e345b78d1..7cbac1565d 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -2,12 +2,12 @@ EXECUTING COMMAND: omdb ["db", "--db-url", "postgresql://root@[::1]:REDACTED_POR termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "--db-url", "junk", "sleds"] termination: Exited(2) @@ -165,25 +165,25 @@ EXECUTING COMMAND: omdb ["db", "sleds"] termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["--dns-server", "[::1]:REDACTED_PORT", "db", "sleds"] termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 6fd84c5eb3..3ebf7046d4 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -8,7 +8,7 @@ external oxide-dev.test 2 create silo: "tes --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "dns", "diff", "external", "2"] termination: Exited(0) @@ -24,7 +24,7 @@ changes: names added: 1, names removed: 0 --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "dns", "names", "external", "2"] termination: Exited(0) @@ -36,7 +36,7 @@ External zone: oxide-dev.test --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-instances"] termination: Exited(0) @@ -49,10 +49,12 @@ Dendrite REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT ExternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 InternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 Nexus REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_PORT sim-b6d65341 +Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 +Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-by-sled"] termination: Exited(0) @@ -67,22 +69,24 @@ sled: sim-b6d65341 (id REDACTED_UUID_REDACTED_UUID_REDACTED) ExternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT InternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT Nexus REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_PORT + Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT + Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "sleds"] termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["mgs", "inventory"] termination: Exited(0) diff --git a/env.sh b/env.sh index 5b1e2b34ac..483a89f597 100644 --- a/env.sh +++ b/env.sh @@ -9,5 +9,6 @@ OMICRON_WS="$(cd $(dirname "${BASH_SOURCE[0]}") && echo $PWD)" export PATH="$OMICRON_WS/out/cockroachdb/bin:$PATH" export PATH="$OMICRON_WS/out/clickhouse:$PATH" export PATH="$OMICRON_WS/out/dendrite-stub/bin:$PATH" +export PATH="$OMICRON_WS/out/mgd/root/opt/oxide/mgd/bin:$PATH" unset OMICRON_WS set +o xtrace diff --git a/illumos-utils/src/destructor.rs b/illumos-utils/src/destructor.rs index e019f2562f..ccc5b15486 100644 --- a/illumos-utils/src/destructor.rs +++ b/illumos-utils/src/destructor.rs @@ -21,7 +21,7 @@ use tokio::sync::mpsc; type SharedBoxFuture = Shared + Send>>>; -/// Future stored within [Destructor]. +/// Future stored within [`Destructor`]. struct ShutdownWaitFuture(SharedBoxFuture>); impl Future for ShutdownWaitFuture { diff --git a/installinator/src/bootstrap.rs b/installinator/src/bootstrap.rs index 2854293d8a..71c76809db 100644 --- a/installinator/src/bootstrap.rs +++ b/installinator/src/bootstrap.rs @@ -20,7 +20,7 @@ use sled_hardware::underlay::BootstrapInterface; use slog::info; use slog::Logger; -const MG_DDM_SERVICE_FMRI: &str = "svc:/system/illumos/mg-ddm"; +const MG_DDM_SERVICE_FMRI: &str = "svc:/oxide/mg-ddm"; const MG_DDM_MANIFEST_PATH: &str = "/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"; // TODO-cleanup The implementation of this function is heavily derived from diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index 9c06aeac77..9bec14664c 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -104,7 +104,7 @@ impl DebugDiscoverOpts { /// Options shared by both [`DebugDiscoverOpts`] and [`InstallOpts`]. #[derive(Debug, Args)] struct DiscoverOpts { - /// The mechanism by which to discover peers: bootstrap or list:[::1]:8000 + /// The mechanism by which to discover peers: bootstrap or `list:[::1]:8000` #[clap(long, default_value_t = DiscoveryMechanism::Bootstrap)] mechanism: DiscoveryMechanism, } diff --git a/internal-dns/src/config.rs b/internal-dns/src/config.rs index e5272cd23a..86dd6e802e 100644 --- a/internal-dns/src/config.rs +++ b/internal-dns/src/config.rs @@ -63,8 +63,9 @@ use crate::names::{ServiceName, DNS_ZONE}; use anyhow::{anyhow, ensure}; use dns_service_client::types::{DnsConfigParams, DnsConfigZone, DnsRecord}; +use omicron_common::api::internal::shared::SwitchLocation; use std::collections::BTreeMap; -use std::net::Ipv6Addr; +use std::net::{Ipv6Addr, SocketAddrV6}; use uuid::Uuid; /// Zones that can be referenced within the internal DNS system. @@ -136,6 +137,8 @@ pub struct DnsConfigBuilder { /// network sleds: BTreeMap, + scrimlets: BTreeMap, + /// set of hosts of type "zone" that have been configured so far, mapping /// each zone's unique uuid to its sole IPv6 address on the control plane /// network @@ -175,6 +178,7 @@ impl DnsConfigBuilder { DnsConfigBuilder { sleds: BTreeMap::new(), zones: BTreeMap::new(), + scrimlets: BTreeMap::new(), service_instances_zones: BTreeMap::new(), service_instances_sleds: BTreeMap::new(), } @@ -205,6 +209,15 @@ impl DnsConfigBuilder { } } + pub fn host_scrimlet( + &mut self, + switch_location: SwitchLocation, + addr: SocketAddrV6, + ) -> anyhow::Result<()> { + self.scrimlets.insert(switch_location, addr); + Ok(()) + } + /// Add a new dendrite host of type "zone" to the configuration /// /// Returns a [`Zone`] that can be used with [`Self::service_backend_zone()`] to @@ -351,6 +364,23 @@ impl DnsConfigBuilder { (zone.dns_name(), vec![DnsRecord::Aaaa(zone_ip)]) }); + let scrimlet_srv_records = + self.scrimlets.clone().into_iter().map(|(location, addr)| { + let srv = DnsRecord::Srv(dns_service_client::types::Srv { + prio: 0, + weight: 0, + port: addr.port(), + target: format!("{location}.scrimlet.{}", DNS_ZONE), + }); + (ServiceName::Scrimlet(location).dns_name(), vec![srv]) + }); + + let scrimlet_aaaa_records = + self.scrimlets.into_iter().map(|(location, addr)| { + let aaaa = DnsRecord::Aaaa(*addr.ip()); + (format!("{location}.scrimlet"), vec![aaaa]) + }); + // Assemble the set of SRV records, which implicitly point back at // zones' AAAA records. let srv_records_zones = self.service_instances_zones.into_iter().map( @@ -399,6 +429,8 @@ impl DnsConfigBuilder { .chain(zone_records) .chain(srv_records_sleds) .chain(srv_records_zones) + .chain(scrimlet_aaaa_records) + .chain(scrimlet_srv_records) .collect(); DnsConfigParams { diff --git a/internal-dns/src/names.rs b/internal-dns/src/names.rs index 44ed9228e2..e0c9b79555 100644 --- a/internal-dns/src/names.rs +++ b/internal-dns/src/names.rs @@ -4,6 +4,7 @@ //! Well-known DNS names and related types for internal DNS (see RFD 248) +use omicron_common::api::internal::shared::SwitchLocation; use uuid::Uuid; /// Name for the control plane DNS zone @@ -32,7 +33,9 @@ pub enum ServiceName { Crucible(Uuid), BoundaryNtp, InternalNtp, - Maghemite, + Maghemite, //TODO change to Dpd - maghemite has several services. + Mgd, + Scrimlet(SwitchLocation), } impl ServiceName { @@ -55,6 +58,8 @@ impl ServiceName { ServiceName::BoundaryNtp => "boundary-ntp", ServiceName::InternalNtp => "internal-ntp", ServiceName::Maghemite => "maghemite", + ServiceName::Mgd => "mgd", + ServiceName::Scrimlet(_) => "scrimlet", } } @@ -76,7 +81,8 @@ impl ServiceName { | ServiceName::CruciblePantry | ServiceName::BoundaryNtp | ServiceName::InternalNtp - | ServiceName::Maghemite => { + | ServiceName::Maghemite + | ServiceName::Mgd => { format!("_{}._tcp", self.service_kind()) } ServiceName::SledAgent(id) => { @@ -85,6 +91,9 @@ impl ServiceName { ServiceName::Crucible(id) => { format!("_{}._tcp.{}", self.service_kind(), id) } + ServiceName::Scrimlet(location) => { + format!("_{location}._scrimlet._tcp") + } } } diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 3de6dac7c0..323386ba25 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -22,6 +22,7 @@ crucible-agent-client.workspace = true crucible-pantry-client.workspace = true dns-service-client.workspace = true dpd-client.workspace = true +mg-admin-client.workspace = true dropshot.workspace = true fatfs.workspace = true futures.workspace = true diff --git a/nexus/db-macros/src/lookup.rs b/nexus/db-macros/src/lookup.rs index 38cab15e30..f2362f5bc5 100644 --- a/nexus/db-macros/src/lookup.rs +++ b/nexus/db-macros/src/lookup.rs @@ -15,7 +15,7 @@ use std::ops::Deref; // INPUT (arguments to the macro) // -/// Arguments for [`lookup_resource!`] +/// Arguments for [`super::lookup_resource!`] // NOTE: this is only "pub" for the `cargo doc` link on [`lookup_resource!`]. #[derive(serde::Deserialize)] pub struct Input { @@ -167,7 +167,7 @@ impl Resource { // MACRO IMPLEMENTATION // -/// Implementation of [`lookup_resource!`] +/// Implementation of [`super::lookup_resource!`] pub fn lookup_resource( raw_input: TokenStream, ) -> Result { diff --git a/nexus/db-model/src/bgp.rs b/nexus/db-model/src/bgp.rs index 532b9cce36..cc9ebfb4f5 100644 --- a/nexus/db-model/src/bgp.rs +++ b/nexus/db-model/src/bgp.rs @@ -6,8 +6,10 @@ use crate::schema::{bgp_announce_set, bgp_announcement, bgp_config}; use crate::SqlU32; use db_macros::Resource; use ipnetwork::IpNetwork; +use nexus_types::external_api::params; use nexus_types::identity::Resource; use omicron_common::api::external; +use omicron_common::api::external::IdentityMetadataCreateParams; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -26,6 +28,7 @@ pub struct BgpConfig { #[diesel(embed)] pub identity: BgpConfigIdentity, pub asn: SqlU32, + pub bgp_announce_set_id: Uuid, pub vrf: Option, } @@ -39,6 +42,26 @@ impl Into for BgpConfig { } } +impl BgpConfig { + pub fn from_config_create( + c: ¶ms::BgpConfigCreate, + bgp_announce_set_id: Uuid, + ) -> BgpConfig { + BgpConfig { + identity: BgpConfigIdentity::new( + Uuid::new_v4(), + IdentityMetadataCreateParams { + name: c.identity.name.clone(), + description: c.identity.description.clone(), + }, + ), + asn: c.asn.into(), + bgp_announce_set_id, + vrf: c.vrf.as_ref().map(|x| x.to_string()), + } + } +} + #[derive( Queryable, Insertable, @@ -55,6 +78,20 @@ pub struct BgpAnnounceSet { pub identity: BgpAnnounceSetIdentity, } +impl From for BgpAnnounceSet { + fn from(x: params::BgpAnnounceSetCreate) -> BgpAnnounceSet { + BgpAnnounceSet { + identity: BgpAnnounceSetIdentity::new( + Uuid::new_v4(), + IdentityMetadataCreateParams { + name: x.identity.name.clone(), + description: x.identity.description.clone(), + }, + ), + } + } +} + impl Into for BgpAnnounceSet { fn into(self) -> external::BgpAnnounceSet { external::BgpAnnounceSet { identity: self.identity() } diff --git a/nexus/db-model/src/bootstore.rs b/nexus/db-model/src/bootstore.rs new file mode 100644 index 0000000000..38afd37f54 --- /dev/null +++ b/nexus/db-model/src/bootstore.rs @@ -0,0 +1,13 @@ +use crate::schema::bootstore_keys; +use serde::{Deserialize, Serialize}; + +pub const NETWORK_KEY: &str = "network_key"; + +#[derive( + Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, +)] +#[diesel(table_name = bootstore_keys)] +pub struct BootstoreKeys { + pub key: String, + pub generation: i64, +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index f1447fc503..f399605f55 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -12,6 +12,7 @@ extern crate newtype_derive; mod address_lot; mod bgp; mod block_size; +mod bootstore; mod bytecount; mod certificate; mod collection; @@ -100,6 +101,7 @@ pub use self::unsigned::*; pub use address_lot::*; pub use bgp::*; pub use block_size::*; +pub use bootstore::*; pub use bytecount::*; pub use certificate::*; pub use collection::*; diff --git a/nexus/db-model/src/rack.rs b/nexus/db-model/src/rack.rs index 0f1ef2a853..580ec155b4 100644 --- a/nexus/db-model/src/rack.rs +++ b/nexus/db-model/src/rack.rs @@ -4,6 +4,7 @@ use crate::schema::rack; use db_macros::Asset; +use ipnetwork::IpNetwork; use nexus_types::{external_api::views, identity::Asset}; use uuid::Uuid; @@ -15,6 +16,7 @@ pub struct Rack { pub identity: RackIdentity, pub initialized: bool, pub tuf_base_url: Option, + pub rack_subnet: Option, } impl Rack { @@ -23,6 +25,7 @@ impl Rack { identity: RackIdentity::new(id), initialized: false, tuf_base_url: None, + rack_subnet: None, } } } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 9189b6db7b..e079432e5a 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -144,6 +144,8 @@ table! { lldp_service_config_id -> Uuid, link_name -> Text, mtu -> Int4, + fec -> crate::SwitchLinkFecEnum, + speed -> crate::SwitchLinkSpeedEnum, } } @@ -188,7 +190,7 @@ table! { } table! { - switch_port_settings_route_config (port_settings_id, interface_name, dst, gw, vid) { + switch_port_settings_route_config (port_settings_id, interface_name, dst, gw) { port_settings_id -> Uuid, interface_name -> Text, dst -> Inet, @@ -200,10 +202,14 @@ table! { table! { switch_port_settings_bgp_peer_config (port_settings_id, interface_name, addr) { port_settings_id -> Uuid, - bgp_announce_set_id -> Uuid, bgp_config_id -> Uuid, interface_name -> Text, addr -> Inet, + hold_time -> Int8, + idle_hold_time -> Int8, + delay_open -> Int8, + connect_retry -> Int8, + keepalive -> Int8, } } @@ -216,6 +222,7 @@ table! { time_modified -> Timestamptz, time_deleted -> Nullable, asn -> Int8, + bgp_announce_set_id -> Uuid, vrf -> Nullable, } } @@ -673,6 +680,7 @@ table! { time_modified -> Timestamptz, initialized -> Bool, tuf_base_url -> Nullable, + rack_subnet -> Nullable, } } @@ -1132,6 +1140,13 @@ table! { } } +table! { + bootstore_keys (key, generation) { + key -> Text, + generation -> Int8, + } +} + table! { db_metadata (singleton) { singleton -> Bool, @@ -1147,7 +1162,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(7, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(8, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/nexus/db-model/src/service_kind.rs b/nexus/db-model/src/service_kind.rs index c2598434d5..4210c3ee20 100644 --- a/nexus/db-model/src/service_kind.rs +++ b/nexus/db-model/src/service_kind.rs @@ -30,6 +30,7 @@ impl_enum_type!( Oximeter => b"oximeter" Tfport => b"tfport" Ntp => b"ntp" + Mgd => b"mgd" ); impl TryFrom for ServiceUsingCertificate { @@ -88,6 +89,7 @@ impl From for ServiceKind { | internal_api::params::ServiceKind::InternalNtp => { ServiceKind::Ntp } + internal_api::params::ServiceKind::Mgd => ServiceKind::Mgd, } } } diff --git a/nexus/db-model/src/switch_interface.rs b/nexus/db-model/src/switch_interface.rs index 9ac7e4323a..f0c4b91de6 100644 --- a/nexus/db-model/src/switch_interface.rs +++ b/nexus/db-model/src/switch_interface.rs @@ -64,7 +64,14 @@ impl Into for DbSwitchInterfaceKind { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_vlan_interface_config)] pub struct SwitchVlanInterfaceConfig { diff --git a/nexus/db-model/src/switch_port.rs b/nexus/db-model/src/switch_port.rs index e9c0697450..f6df50ef97 100644 --- a/nexus/db-model/src/switch_port.rs +++ b/nexus/db-model/src/switch_port.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::impl_enum_type; use crate::schema::{ lldp_config, lldp_service_config, switch_port, switch_port_settings, switch_port_settings_address_config, switch_port_settings_bgp_peer_config, @@ -11,11 +10,14 @@ use crate::schema::{ switch_port_settings_port_config, switch_port_settings_route_config, }; use crate::SqlU16; +use crate::{impl_enum_type, SqlU32}; use db_macros::Resource; +use diesel::AsChangeset; use ipnetwork::IpNetwork; use nexus_types::external_api::params; use nexus_types::identity::Resource; use omicron_common::api::external; +use omicron_common::api::internal::shared::{PortFec, PortSpeed}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -42,6 +44,110 @@ impl_enum_type!( Sfp28x4 => b"Sfp28x4" ); +impl_enum_type!( + #[derive(SqlType, Debug, Clone, Copy)] + #[diesel(postgres_type(name = "switch_link_fec"))] + pub struct SwitchLinkFecEnum; + + #[derive( + Clone, + Copy, + Debug, + AsExpression, + FromSqlRow, + PartialEq, + Serialize, + Deserialize + )] + #[diesel(sql_type = SwitchLinkFecEnum)] + pub enum SwitchLinkFec; + + Firecode => b"Firecode" + None => b"None" + Rs => b"Rs" +); + +impl_enum_type!( + #[derive(SqlType, Debug, Clone, Copy)] + #[diesel(postgres_type(name = "switch_link_speed"))] + pub struct SwitchLinkSpeedEnum; + + #[derive( + Clone, + Copy, + Debug, + AsExpression, + FromSqlRow, + PartialEq, + Serialize, + Deserialize + )] + #[diesel(sql_type = SwitchLinkSpeedEnum)] + pub enum SwitchLinkSpeed; + + Speed0G => b"0G" + Speed1G => b"1G" + Speed10G => b"10G" + Speed25G => b"25G" + Speed40G => b"40G" + Speed50G => b"50G" + Speed100G => b"100G" + Speed200G => b"200G" + Speed400G => b"400G" +); + +impl From for PortFec { + fn from(value: SwitchLinkFec) -> Self { + match value { + SwitchLinkFec::Firecode => PortFec::Firecode, + SwitchLinkFec::None => PortFec::None, + SwitchLinkFec::Rs => PortFec::Rs, + } + } +} + +impl From for SwitchLinkFec { + fn from(value: params::LinkFec) -> Self { + match value { + params::LinkFec::Firecode => SwitchLinkFec::Firecode, + params::LinkFec::None => SwitchLinkFec::None, + params::LinkFec::Rs => SwitchLinkFec::Rs, + } + } +} + +impl From for PortSpeed { + fn from(value: SwitchLinkSpeed) -> Self { + match value { + SwitchLinkSpeed::Speed0G => PortSpeed::Speed0G, + SwitchLinkSpeed::Speed1G => PortSpeed::Speed1G, + SwitchLinkSpeed::Speed10G => PortSpeed::Speed10G, + SwitchLinkSpeed::Speed25G => PortSpeed::Speed25G, + SwitchLinkSpeed::Speed40G => PortSpeed::Speed40G, + SwitchLinkSpeed::Speed50G => PortSpeed::Speed50G, + SwitchLinkSpeed::Speed100G => PortSpeed::Speed100G, + SwitchLinkSpeed::Speed200G => PortSpeed::Speed200G, + SwitchLinkSpeed::Speed400G => PortSpeed::Speed400G, + } + } +} + +impl From for SwitchLinkSpeed { + fn from(value: params::LinkSpeed) -> Self { + match value { + params::LinkSpeed::Speed0G => SwitchLinkSpeed::Speed0G, + params::LinkSpeed::Speed1G => SwitchLinkSpeed::Speed1G, + params::LinkSpeed::Speed10G => SwitchLinkSpeed::Speed10G, + params::LinkSpeed::Speed25G => SwitchLinkSpeed::Speed25G, + params::LinkSpeed::Speed40G => SwitchLinkSpeed::Speed40G, + params::LinkSpeed::Speed50G => SwitchLinkSpeed::Speed50G, + params::LinkSpeed::Speed100G => SwitchLinkSpeed::Speed100G, + params::LinkSpeed::Speed200G => SwitchLinkSpeed::Speed200G, + params::LinkSpeed::Speed400G => SwitchLinkSpeed::Speed400G, + } + } +} + impl From for SwitchPortGeometry { fn from(g: params::SwitchPortGeometry) -> Self { match g { @@ -225,7 +331,14 @@ impl Into for SwitchPortConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_link_config)] pub struct SwitchPortLinkConfig { @@ -233,6 +346,8 @@ pub struct SwitchPortLinkConfig { pub lldp_service_config_id: Uuid, pub link_name: String, pub mtu: SqlU16, + pub fec: SwitchLinkFec, + pub speed: SwitchLinkSpeed, } impl SwitchPortLinkConfig { @@ -241,11 +356,15 @@ impl SwitchPortLinkConfig { lldp_service_config_id: Uuid, link_name: String, mtu: u16, + fec: SwitchLinkFec, + speed: SwitchLinkSpeed, ) -> Self { Self { port_settings_id, lldp_service_config_id, link_name, + fec, + speed, mtu: mtu.into(), } } @@ -263,7 +382,14 @@ impl Into for SwitchPortLinkConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = lldp_service_config)] pub struct LldpServiceConfig { @@ -321,7 +447,14 @@ impl Into for LldpConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_interface_config)] pub struct SwitchInterfaceConfig { @@ -362,7 +495,14 @@ impl Into for SwitchInterfaceConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_route_config)] pub struct SwitchPortRouteConfig { @@ -398,31 +538,51 @@ impl Into for SwitchPortRouteConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_bgp_peer_config)] pub struct SwitchPortBgpPeerConfig { pub port_settings_id: Uuid, - pub bgp_announce_set_id: Uuid, pub bgp_config_id: Uuid, pub interface_name: String, pub addr: IpNetwork, + pub hold_time: SqlU32, + pub idle_hold_time: SqlU32, + pub delay_open: SqlU32, + pub connect_retry: SqlU32, + pub keepalive: SqlU32, } impl SwitchPortBgpPeerConfig { + #[allow(clippy::too_many_arguments)] pub fn new( port_settings_id: Uuid, - bgp_announce_set_id: Uuid, bgp_config_id: Uuid, interface_name: String, addr: IpNetwork, + hold_time: SqlU32, + idle_hold_time: SqlU32, + delay_open: SqlU32, + connect_retry: SqlU32, + keepalive: SqlU32, ) -> Self { Self { port_settings_id, - bgp_announce_set_id, bgp_config_id, interface_name, addr, + hold_time, + idle_hold_time, + delay_open, + connect_retry, + keepalive, } } } @@ -431,7 +591,6 @@ impl Into for SwitchPortBgpPeerConfig { fn into(self) -> external::SwitchPortBgpPeerConfig { external::SwitchPortBgpPeerConfig { port_settings_id: self.port_settings_id, - bgp_announce_set_id: self.bgp_announce_set_id, bgp_config_id: self.bgp_config_id, interface_name: self.interface_name.clone(), addr: self.addr.ip(), @@ -440,7 +599,14 @@ impl Into for SwitchPortBgpPeerConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_address_config)] pub struct SwitchPortAddressConfig { diff --git a/nexus/db-queries/src/db/datastore/bgp.rs b/nexus/db-queries/src/db/datastore/bgp.rs new file mode 100644 index 0000000000..ff314a2564 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/bgp.rs @@ -0,0 +1,351 @@ +use super::DataStore; +use crate::context::OpContext; +use crate::db; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::error::TransactionError; +use crate::db::model::Name; +use crate::db::model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig}; +use crate::db::pagination::paginated; +use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; +use chrono::Utc; +use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; +use nexus_types::external_api::params; +use nexus_types::identity::Resource; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::{ + CreateResult, DeleteResult, Error, ListResultVec, LookupResult, NameOrId, + ResourceType, +}; +use ref_cast::RefCast; +use uuid::Uuid; + +impl DataStore { + pub async fn bgp_config_set( + &self, + opctx: &OpContext, + config: ¶ms::BgpConfigCreate, + ) -> CreateResult { + use db::schema::bgp_config::dsl; + use db::schema::{ + bgp_announce_set, bgp_announce_set::dsl as announce_set_dsl, + }; + let pool = self.pool_connection_authorized(opctx).await?; + + pool.transaction_async(|conn| async move { + let id: Uuid = match &config.bgp_announce_set_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::time_deleted.is_null()) + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => *id, + }; + + let config = BgpConfig::from_config_create(config, id); + + let result = diesel::insert_into(dsl::bgp_config) + .values(config.clone()) + .returning(BgpConfig::as_returning()) + .get_result_async(&conn) + .await?; + Ok(result) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn bgp_config_delete( + &self, + opctx: &OpContext, + sel: ¶ms::BgpConfigSelector, + ) -> DeleteResult { + use db::schema::bgp_config; + use db::schema::bgp_config::dsl as bgp_config_dsl; + + use db::schema::switch_port_settings_bgp_peer_config as sps_bgp_peer_config; + use db::schema::switch_port_settings_bgp_peer_config::dsl as sps_bgp_peer_config_dsl; + + #[derive(Debug)] + enum BgpConfigDeleteError { + ConfigInUse, + } + type TxnError = TransactionError; + + let pool = self.pool_connection_authorized(opctx).await?; + pool.transaction_async(|conn| async move { + let name_or_id = sel.name_or_id.clone(); + + let id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => { + bgp_config_dsl::bgp_config + .filter(bgp_config::name.eq(name.to_string())) + .select(bgp_config::id) + .limit(1) + .first_async::(&conn) + .await? + } + }; + + let count = + sps_bgp_peer_config_dsl::switch_port_settings_bgp_peer_config + .filter(sps_bgp_peer_config::bgp_config_id.eq(id)) + .count() + .execute_async(&conn) + .await?; + + if count > 0 { + return Err(TxnError::CustomError( + BgpConfigDeleteError::ConfigInUse, + )); + } + + diesel::update(bgp_config_dsl::bgp_config) + .filter(bgp_config_dsl::id.eq(id)) + .set(bgp_config_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; + + Ok(()) + }) + .await + .map_err(|e| match e { + TxnError::CustomError(BgpConfigDeleteError::ConfigInUse) => { + Error::invalid_request("BGP config in use") + } + TxnError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + + pub async fn bgp_config_get( + &self, + opctx: &OpContext, + name_or_id: &NameOrId, + ) -> LookupResult { + use db::schema::bgp_config; + use db::schema::bgp_config::dsl; + let pool = self.pool_connection_authorized(opctx).await?; + + let name_or_id = name_or_id.clone(); + + let config = match name_or_id { + NameOrId::Name(name) => dsl::bgp_config + .filter(bgp_config::name.eq(name.to_string())) + .select(BgpConfig::as_select()) + .limit(1) + .first_async::(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), + NameOrId::Id(id) => dsl::bgp_config + .filter(bgp_config::id.eq(id)) + .select(BgpConfig::as_select()) + .limit(1) + .first_async::(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), + }?; + + Ok(config) + } + + pub async fn bgp_config_list( + &self, + opctx: &OpContext, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::bgp_config::dsl; + + let pool = self.pool_connection_authorized(opctx).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::bgp_config, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::bgp_config, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::time_deleted.is_null()) + .select(BgpConfig::as_select()) + .load_async(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn bgp_announce_list( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> ListResultVec { + use db::schema::{ + bgp_announce_set, bgp_announce_set::dsl as announce_set_dsl, + bgp_announcement::dsl as announce_dsl, + }; + + #[derive(Debug)] + enum BgpAnnounceListError { + AnnounceSetNotFound(Name), + } + type TxnError = TransactionError; + + let pool = self.pool_connection_authorized(opctx).await?; + pool.transaction_async(|conn| async move { + let name_or_id = sel.name_or_id.clone(); + + let announce_id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::time_deleted.is_null()) + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|_| { + TxnError::CustomError( + BgpAnnounceListError::AnnounceSetNotFound( + Name::from(name.clone()), + ), + ) + })?, + }; + + let result = announce_dsl::bgp_announcement + .filter(announce_dsl::announce_set_id.eq(announce_id)) + .select(BgpAnnouncement::as_select()) + .load_async(&conn) + .await?; + + Ok(result) + }) + .await + .map_err(|e| match e { + TxnError::CustomError( + BgpAnnounceListError::AnnounceSetNotFound(name), + ) => Error::not_found_by_name(ResourceType::BgpAnnounceSet, &name), + TxnError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + + pub async fn bgp_create_announce_set( + &self, + opctx: &OpContext, + announce: ¶ms::BgpAnnounceSetCreate, + ) -> CreateResult<(BgpAnnounceSet, Vec)> { + use db::schema::bgp_announce_set::dsl as announce_set_dsl; + use db::schema::bgp_announcement::dsl as bgp_announcement_dsl; + + let pool = self.pool_connection_authorized(opctx).await?; + pool.transaction_async(|conn| async move { + let bas: BgpAnnounceSet = announce.clone().into(); + + let db_as: BgpAnnounceSet = + diesel::insert_into(announce_set_dsl::bgp_announce_set) + .values(bas.clone()) + .returning(BgpAnnounceSet::as_returning()) + .get_result_async::(&conn) + .await?; + + let mut db_annoucements = Vec::new(); + for a in &announce.announcement { + let an = BgpAnnouncement { + announce_set_id: db_as.id(), + address_lot_block_id: bas.identity.id, + network: a.network.into(), + }; + let an = + diesel::insert_into(bgp_announcement_dsl::bgp_announcement) + .values(an.clone()) + .returning(BgpAnnouncement::as_returning()) + .get_result_async::(&conn) + .await?; + db_annoucements.push(an); + } + + Ok((db_as, db_annoucements)) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn bgp_delete_announce_set( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> DeleteResult { + use db::schema::bgp_announce_set; + use db::schema::bgp_announce_set::dsl as announce_set_dsl; + use db::schema::bgp_announcement::dsl as bgp_announcement_dsl; + + use db::schema::bgp_config; + use db::schema::bgp_config::dsl as bgp_config_dsl; + + #[derive(Debug)] + enum BgpAnnounceSetDeleteError { + AnnounceSetInUse, + } + type TxnError = TransactionError; + + let pool = self.pool_connection_authorized(opctx).await?; + let name_or_id = sel.name_or_id.clone(); + + pool.transaction_async(|conn| async move { + let id: Uuid = match name_or_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => id, + }; + + let count = bgp_config_dsl::bgp_config + .filter(bgp_config::bgp_announce_set_id.eq(id)) + .count() + .execute_async(&conn) + .await?; + + if count > 0 { + return Err(TxnError::CustomError( + BgpAnnounceSetDeleteError::AnnounceSetInUse, + )); + } + + diesel::update(announce_set_dsl::bgp_announce_set) + .filter(announce_set_dsl::id.eq(id)) + .set(announce_set_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; + + diesel::delete(bgp_announcement_dsl::bgp_announcement) + .filter(bgp_announcement_dsl::announce_set_id.eq(id)) + .execute_async(&conn) + .await?; + + Ok(()) + }) + .await + .map_err(|e| match e { + TxnError::CustomError( + BgpAnnounceSetDeleteError::AnnounceSetInUse, + ) => Error::invalid_request("BGP announce set in use"), + TxnError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } +} diff --git a/nexus/db-queries/src/db/datastore/bootstore.rs b/nexus/db-queries/src/db/datastore/bootstore.rs new file mode 100644 index 0000000000..44f7a2036e --- /dev/null +++ b/nexus/db-queries/src/db/datastore/bootstore.rs @@ -0,0 +1,37 @@ +use super::DataStore; +use crate::context::OpContext; +use crate::db; +use crate::db::error::{public_error_from_diesel, ErrorHandler}; +use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::ExpressionMethods; +use diesel::SelectableHelper; +use nexus_db_model::BootstoreKeys; +use omicron_common::api::external::LookupResult; + +impl DataStore { + pub async fn bump_bootstore_generation( + &self, + opctx: &OpContext, + key: String, + ) -> LookupResult { + use db::schema::bootstore_keys; + use db::schema::bootstore_keys::dsl; + + let conn = self.pool_connection_authorized(opctx).await?; + + let bks = diesel::insert_into(dsl::bootstore_keys) + .values(BootstoreKeys { + key: key.clone(), + generation: 2, // RSS starts with a generation of 1 + }) + .on_conflict(bootstore_keys::key) + .do_update() + .set(bootstore_keys::generation.eq(dsl::generation + 1)) + .returning(BootstoreKeys::as_returning()) + .get_result_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(bks.generation) + } +} diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index a77e20647a..f5283e263e 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -48,6 +48,8 @@ use std::sync::Arc; use uuid::Uuid; mod address_lot; +mod bgp; +mod bootstore; mod certificate; mod console_session; mod dataset; diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index f5f7524aab..ae982d86f8 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -32,6 +32,7 @@ use chrono::Utc; use diesel::prelude::*; use diesel::result::Error as DieselError; use diesel::upsert::excluded; +use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::DnsZone; use nexus_db_model::ExternalIp; @@ -61,6 +62,7 @@ use uuid::Uuid; #[derive(Clone)] pub struct RackInit { pub rack_id: Uuid, + pub rack_subnet: IpNetwork, pub services: Vec, pub datasets: Vec, pub service_ip_pool_ranges: Vec, @@ -190,6 +192,28 @@ impl DataStore { }) } + pub async fn update_rack_subnet( + &self, + opctx: &OpContext, + rack: &Rack, + ) -> Result<(), Error> { + debug!( + opctx.log, + "updating rack subnet for rack {} to {:#?}", + rack.id(), + rack.rack_subnet + ); + use db::schema::rack::dsl; + diesel::update(dsl::rack) + .filter(dsl::id.eq(rack.id())) + .set(dsl::rack_subnet.eq(rack.rack_subnet)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } + // The following methods which return a `TxnError` take a `conn` parameter // which comes from the transaction created in `rack_set_initialized`. @@ -681,6 +705,7 @@ mod test { fn default() -> Self { RackInit { rack_id: Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap(), + rack_subnet: nexus_test_utils::RACK_SUBNET.parse().unwrap(), services: vec![], datasets: vec![], service_ip_pool_ranges: vec![], diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index 45be594be6..f2126bd968 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -97,41 +97,79 @@ pub struct SwitchPortSettingsGroupCreateResult { } impl DataStore { - // port settings + pub async fn switch_port_settings_exist( + &self, + opctx: &OpContext, + name: Name, + ) -> LookupResult { + use db::schema::switch_port_settings::{ + self, dsl as port_settings_dsl, + }; + + let pool = self.pool_connection_authorized(opctx).await?; + + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::name.eq(name)) + .select(switch_port_settings::id) + .limit(1) + .first_async::(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn switch_ports_using_settings( + &self, + opctx: &OpContext, + switch_port_settings_id: Uuid, + ) -> LookupResult> { + use db::schema::switch_port::{self, dsl}; + + let pool = self.pool_connection_authorized(opctx).await?; + + dsl::switch_port + .filter(switch_port::port_settings_id.eq(switch_port_settings_id)) + .select((switch_port::id, switch_port::port_name)) + .load_async(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } pub async fn switch_port_settings_create( &self, opctx: &OpContext, params: ¶ms::SwitchPortSettingsCreate, ) -> CreateResult { - use db::schema::address_lot::dsl as address_lot_dsl; - use db::schema::bgp_announce_set::dsl as bgp_announce_set_dsl; - use db::schema::bgp_config::dsl as bgp_config_dsl; - use db::schema::lldp_service_config::dsl as lldp_config_dsl; - use db::schema::switch_port_settings::dsl as port_settings_dsl; - use db::schema::switch_port_settings_address_config::dsl as address_config_dsl; - use db::schema::switch_port_settings_bgp_peer_config::dsl as bgp_peer_dsl; - use db::schema::switch_port_settings_interface_config::dsl as interface_config_dsl; - use db::schema::switch_port_settings_link_config::dsl as link_config_dsl; - use db::schema::switch_port_settings_port_config::dsl as port_config_dsl; - use db::schema::switch_port_settings_route_config::dsl as route_config_dsl; - use db::schema::switch_vlan_interface_config::dsl as vlan_config_dsl; + use db::schema::{ + address_lot::dsl as address_lot_dsl, + //XXX ANNOUNCE bgp_announce_set::dsl as bgp_announce_set_dsl, + bgp_config::dsl as bgp_config_dsl, + lldp_service_config::dsl as lldp_config_dsl, + switch_port_settings::dsl as port_settings_dsl, + switch_port_settings_address_config::dsl as address_config_dsl, + switch_port_settings_bgp_peer_config::dsl as bgp_peer_dsl, + switch_port_settings_interface_config::dsl as interface_config_dsl, + switch_port_settings_link_config::dsl as link_config_dsl, + switch_port_settings_port_config::dsl as port_config_dsl, + switch_port_settings_route_config::dsl as route_config_dsl, + switch_vlan_interface_config::dsl as vlan_config_dsl, + }; #[derive(Debug)] enum SwitchPortSettingsCreateError { AddressLotNotFound, - BgpAnnounceSetNotFound, + //XXX ANNOUNCE BgpAnnounceSetNotFound, BgpConfigNotFound, ReserveBlock(ReserveBlockError), } type TxnError = TransactionError; + type SpsCreateError = SwitchPortSettingsCreateError; let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage conn.transaction_async(|conn| async move { - // create the top level port settings object let port_settings = SwitchPortSettings::new(¶ms.identity); let db_port_settings: SwitchPortSettings = @@ -189,6 +227,8 @@ impl DataStore { lldp_svc_config.id, link_name.clone(), c.mtu, + c.fec.into(), + c.speed.into(), )); } result.link_lldp = @@ -260,33 +300,6 @@ impl DataStore { let mut bgp_peer_config = Vec::new(); for (interface_name, p) in ¶ms.bgp_peers { - - // add the bgp peer - // TODO this requires pluming in the API to create - // - bgp configs - // - announce sets - // - announcements - - use db::schema::bgp_announce_set; - let announce_set_id = match &p.bgp_announce_set { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - bgp_announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::time_deleted.is_null()) - .filter(bgp_announce_set::name.eq(name)) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpAnnounceSetNotFound, - ) - })? - } - }; - use db::schema::bgp_config; let bgp_config_id = match &p.bgp_config { NameOrId::Id(id) => *id, @@ -309,10 +322,14 @@ impl DataStore { bgp_peer_config.push(SwitchPortBgpPeerConfig::new( psid, - announce_set_id, bgp_config_id, interface_name.clone(), p.addr.into(), + p.hold_time.into(), + p.idle_hold_time.into(), + p.delay_open.into(), + p.connect_retry.into(), + p.keepalive.into(), )); } @@ -389,16 +406,10 @@ impl DataStore { }) .await .map_err(|e| match e { - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpAnnounceSetNotFound) => { - Error::invalid_request("BGP announce set not found") - } - TxnError::CustomError( - SwitchPortSettingsCreateError::AddressLotNotFound) => { + TxnError::CustomError(SpsCreateError::AddressLotNotFound) => { Error::invalid_request("AddressLot not found") } - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpConfigNotFound) => { + TxnError::CustomError(SpsCreateError::BgpConfigNotFound) => { Error::invalid_request("BGP config not found") } TxnError::CustomError( @@ -475,30 +486,31 @@ impl DataStore { .await?; // delete the port config object - use db::schema::switch_port_settings_port_config; - use db::schema::switch_port_settings_port_config::dsl as port_config_dsl; + use db::schema::switch_port_settings_port_config::{ + self as sps_port_config, dsl as port_config_dsl, + }; diesel::delete(port_config_dsl::switch_port_settings_port_config) - .filter(switch_port_settings_port_config::port_settings_id.eq(id)) + .filter(sps_port_config::port_settings_id.eq(id)) .execute_async(&conn) .await?; // delete the link configs - use db::schema::switch_port_settings_link_config; - use db::schema::switch_port_settings_link_config::dsl as link_config_dsl; + use db::schema::switch_port_settings_link_config::{ + self as sps_link_config, dsl as link_config_dsl, + }; let links: Vec = diesel::delete( link_config_dsl::switch_port_settings_link_config ) .filter( - switch_port_settings_link_config::port_settings_id.eq(id) + sps_link_config::port_settings_id.eq(id) ) .returning(SwitchPortLinkConfig::as_returning()) .get_results_async(&conn) .await?; // delete lldp configs - use db::schema::lldp_service_config; - use db::schema::lldp_service_config::dsl as lldp_config_dsl; + use db::schema::lldp_service_config::{self, dsl as lldp_config_dsl}; let lldp_svc_ids: Vec = links .iter() .map(|link| link.lldp_service_config_id) @@ -509,26 +521,25 @@ impl DataStore { .await?; // delete interface configs - use db::schema::switch_port_settings_interface_config; - use db::schema::switch_port_settings_interface_config::dsl - as interface_config_dsl; + use db::schema::switch_port_settings_interface_config::{ + self as sps_interface_config, dsl as interface_config_dsl, + }; let interfaces: Vec = diesel::delete( interface_config_dsl::switch_port_settings_interface_config ) .filter( - switch_port_settings_interface_config::port_settings_id.eq( - id - ) + sps_interface_config::port_settings_id.eq(id) ) .returning(SwitchInterfaceConfig::as_returning()) .get_results_async(&conn) .await?; // delete any vlan interfaces - use db::schema::switch_vlan_interface_config; - use db::schema::switch_vlan_interface_config::dsl as vlan_config_dsl; + use db::schema::switch_vlan_interface_config::{ + self, dsl as vlan_config_dsl, + }; let interface_ids: Vec = interfaces .iter() .map(|interface| interface.id) @@ -566,22 +577,26 @@ impl DataStore { .await?; // delete address configs - use db::schema::switch_port_settings_address_config as address_config; - use db::schema::switch_port_settings_address_config::dsl - as address_config_dsl; + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; - let ps = diesel::delete(address_config_dsl::switch_port_settings_address_config) - .filter(address_config::port_settings_id.eq(id)) - .returning(SwitchPortAddressConfig::as_returning()) - .get_result_async(&conn) - .await?; + let port_settings_addrs = diesel::delete( + address_config_dsl::switch_port_settings_address_config, + ) + .filter(address_config::port_settings_id.eq(id)) + .returning(SwitchPortAddressConfig::as_returning()) + .get_results_async(&conn) + .await?; use db::schema::address_lot_rsvd_block::dsl as rsvd_block_dsl; - diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) - .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) - .execute_async(&conn) - .await?; + for ps in &port_settings_addrs { + diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) + .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) + .execute_async(&conn) + .await?; + } Ok(()) }) @@ -650,10 +665,10 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage conn.transaction_async(|conn| async move { - // get the top level port settings object - use db::schema::switch_port_settings::dsl as port_settings_dsl; - use db::schema::switch_port_settings; + use db::schema::switch_port_settings::{ + self, dsl as port_settings_dsl, + }; let id = match name_or_id { NameOrId::Id(id) => *id, @@ -668,23 +683,27 @@ impl DataStore { .await .map_err(|_| { TxnError::CustomError( - SwitchPortSettingsGetError::NotFound(name.clone()) + SwitchPortSettingsGetError::NotFound( + name.clone(), + ), ) })? } }; - let settings: SwitchPortSettings = port_settings_dsl::switch_port_settings - .filter(switch_port_settings::time_deleted.is_null()) - .filter(switch_port_settings::id.eq(id)) - .select(SwitchPortSettings::as_select()) - .limit(1) - .first_async::(&conn) - .await?; + let settings: SwitchPortSettings = + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::id.eq(id)) + .select(SwitchPortSettings::as_select()) + .limit(1) + .first_async::(&conn) + .await?; // get the port config - use db::schema::switch_port_settings_port_config::dsl as port_config_dsl; - use db::schema::switch_port_settings_port_config as port_config; + use db::schema::switch_port_settings_port_config::{ + self as port_config, dsl as port_config_dsl, + }; let port: SwitchPortConfig = port_config_dsl::switch_port_settings_port_config .filter(port_config::port_settings_id.eq(id)) @@ -694,11 +713,13 @@ impl DataStore { .await?; // initialize result - let mut result = SwitchPortSettingsCombinedResult::new(settings, port); + let mut result = + SwitchPortSettingsCombinedResult::new(settings, port); // get the link configs - use db::schema::switch_port_settings_link_config::dsl as link_config_dsl; - use db::schema::switch_port_settings_link_config as link_config; + use db::schema::switch_port_settings_link_config::{ + self as link_config, dsl as link_config_dsl, + }; result.links = link_config_dsl::switch_port_settings_link_config .filter(link_config::port_settings_id.eq(id)) @@ -706,25 +727,25 @@ impl DataStore { .load_async::(&conn) .await?; - let lldp_svc_ids: Vec = result.links + let lldp_svc_ids: Vec = result + .links .iter() .map(|link| link.lldp_service_config_id) .collect(); - use db::schema::lldp_service_config::dsl as lldp_dsl; use db::schema::lldp_service_config as lldp_config; - result.link_lldp = - lldp_dsl::lldp_service_config - .filter(lldp_config::id.eq_any(lldp_svc_ids)) - .select(LldpServiceConfig::as_select()) - .limit(1) - .load_async::(&conn) - .await?; + use db::schema::lldp_service_config::dsl as lldp_dsl; + result.link_lldp = lldp_dsl::lldp_service_config + .filter(lldp_config::id.eq_any(lldp_svc_ids)) + .select(LldpServiceConfig::as_select()) + .limit(1) + .load_async::(&conn) + .await?; // get the interface configs - use db::schema::switch_port_settings_interface_config::dsl - as interface_config_dsl; - use db::schema::switch_port_settings_interface_config as interface_config; + use db::schema::switch_port_settings_interface_config::{ + self as interface_config, dsl as interface_config_dsl, + }; result.interfaces = interface_config_dsl::switch_port_settings_interface_config @@ -733,37 +754,35 @@ impl DataStore { .load_async::(&conn) .await?; - use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; use db::schema::switch_vlan_interface_config as vlan_config; - let interface_ids: Vec = result.interfaces + use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; + let interface_ids: Vec = result + .interfaces .iter() .map(|interface| interface.id) .collect(); - result.vlan_interfaces = - vlan_dsl::switch_vlan_interface_config - .filter( - vlan_config::interface_config_id.eq_any(interface_ids) - ) + result.vlan_interfaces = vlan_dsl::switch_vlan_interface_config + .filter(vlan_config::interface_config_id.eq_any(interface_ids)) .select(SwitchVlanInterfaceConfig::as_select()) .load_async::(&conn) .await?; - // get the route configs - use db::schema::switch_port_settings_route_config::dsl as route_config_dsl; - use db::schema::switch_port_settings_route_config as route_config; + use db::schema::switch_port_settings_route_config::{ + self as route_config, dsl as route_config_dsl, + }; - result.routes = - route_config_dsl::switch_port_settings_route_config - .filter(route_config::port_settings_id.eq(id)) - .select(SwitchPortRouteConfig::as_select()) - .load_async::(&conn) - .await?; + result.routes = route_config_dsl::switch_port_settings_route_config + .filter(route_config::port_settings_id.eq(id)) + .select(SwitchPortRouteConfig::as_select()) + .load_async::(&conn) + .await?; // get the bgp peer configs - use db::schema::switch_port_settings_bgp_peer_config::dsl as bgp_peer_dsl; - use db::schema::switch_port_settings_bgp_peer_config as bgp_peer; + use db::schema::switch_port_settings_bgp_peer_config::{ + self as bgp_peer, dsl as bgp_peer_dsl, + }; result.bgp_peers = bgp_peer_dsl::switch_port_settings_bgp_peer_config @@ -773,9 +792,9 @@ impl DataStore { .await?; // get the address configs - use db::schema::switch_port_settings_address_config::dsl - as address_config_dsl; - use db::schema::switch_port_settings_address_config as address_config; + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; result.addresses = address_config_dsl::switch_port_settings_address_config @@ -785,14 +804,15 @@ impl DataStore { .await?; Ok(result) - }) .await .map_err(|e| match e { - TxnError::CustomError( - SwitchPortSettingsGetError::NotFound(name)) => { - Error::not_found_by_name(ResourceType::SwitchPortSettings, &name) - } + TxnError::CustomError(SwitchPortSettingsGetError::NotFound( + name, + )) => Error::not_found_by_name( + ResourceType::SwitchPortSettings, + &name, + ), TxnError::Database(e) => match e { DieselError::DatabaseError(_, _) => { let name = name_or_id.to_string(); @@ -803,7 +823,7 @@ impl DataStore { &name, ), ) - }, + } _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) @@ -1083,8 +1103,10 @@ impl DataStore { &self, opctx: &OpContext, ) -> ListResultVec { - use db::schema::switch_port::dsl as switch_port_dsl; - use db::schema::switch_port_settings_route_config::dsl as route_config_dsl; + use db::schema::{ + switch_port::dsl as switch_port_dsl, + switch_port_settings_route_config::dsl as route_config_dsl, + }; switch_port_dsl::switch_port .filter(switch_port_dsl::port_settings_id.is_not_null()) diff --git a/nexus/src/app/bgp.rs b/nexus/src/app/bgp.rs new file mode 100644 index 0000000000..e800d72bdd --- /dev/null +++ b/nexus/src/app/bgp.rs @@ -0,0 +1,162 @@ +use crate::app::authz; +use crate::external_api::params; +use nexus_db_model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig}; +use nexus_db_queries::context::OpContext; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::{ + BgpImportedRouteIpv4, BgpPeerStatus, CreateResult, DeleteResult, Ipv4Net, + ListResultVec, LookupResult, NameOrId, +}; + +impl super::Nexus { + pub async fn bgp_config_set( + &self, + opctx: &OpContext, + config: ¶ms::BgpConfigCreate, + ) -> CreateResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = self.db_datastore.bgp_config_set(opctx, config).await?; + Ok(result) + } + + pub async fn bgp_config_get( + &self, + opctx: &OpContext, + name_or_id: NameOrId, + ) -> LookupResult { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + self.db_datastore.bgp_config_get(opctx, &name_or_id).await + } + + pub async fn bgp_config_list( + &self, + opctx: &OpContext, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + self.db_datastore.bgp_config_list(opctx, pagparams).await + } + + pub async fn bgp_config_delete( + &self, + opctx: &OpContext, + sel: ¶ms::BgpConfigSelector, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = self.db_datastore.bgp_config_delete(opctx, sel).await?; + Ok(result) + } + + pub async fn bgp_create_announce_set( + &self, + opctx: &OpContext, + announce: ¶ms::BgpAnnounceSetCreate, + ) -> CreateResult<(BgpAnnounceSet, Vec)> { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = + self.db_datastore.bgp_create_announce_set(opctx, announce).await?; + Ok(result) + } + + pub async fn bgp_announce_list( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + self.db_datastore.bgp_announce_list(opctx, sel).await + } + + pub async fn bgp_delete_announce_set( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = + self.db_datastore.bgp_delete_announce_set(opctx, sel).await?; + Ok(result) + } + + pub async fn bgp_peer_status( + &self, + opctx: &OpContext, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + let mut result = Vec::new(); + for (switch, client) in &self.mg_clients { + let router_info = match client.inner.get_routers().await { + Ok(result) => result.into_inner(), + Err(e) => { + error!( + self.log, + "failed to get routers from {switch}: {e}" + ); + continue; + } + }; + + for r in &router_info { + for (addr, info) in &r.peers { + let Ok(addr) = addr.parse() else { + continue; + }; + result.push(BgpPeerStatus { + switch: *switch, + addr, + local_asn: r.asn, + remote_asn: info.asn.unwrap_or(0), + state: info.state.into(), + state_duration_millis: info.duration_millis, + }); + } + } + } + Ok(result) + } + + pub async fn bgp_imported_routes_ipv4( + &self, + opctx: &OpContext, + sel: ¶ms::BgpRouteSelector, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + let mut result = Vec::new(); + for (switch, client) in &self.mg_clients { + let imported: Vec = match client + .inner + .get_imported4(&mg_admin_client::types::GetImported4Request { + asn: sel.asn, + }) + .await + { + Ok(result) => result + .into_inner() + .into_iter() + .map(|x| BgpImportedRouteIpv4 { + switch: *switch, + prefix: Ipv4Net( + ipnetwork::Ipv4Network::new( + x.prefix.value, + x.prefix.length, + ) + .unwrap(), + ), + nexthop: x.nexthop, + id: x.id, + }) + .collect(), + Err(e) => { + error!( + self.log, + "failed to get BGP imported from {switch}: {e}" + ); + continue; + } + }; + + result.extend_from_slice(&imported); + } + Ok(result) + } +} diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 23ded83150..7db93a158a 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -20,6 +20,7 @@ use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use omicron_common::address::DENDRITE_PORT; +use omicron_common::address::MGD_PORT; use omicron_common::address::MGS_PORT; use omicron_common::api::external::Error; use omicron_common::api::internal::shared::SwitchLocation; @@ -34,6 +35,7 @@ use uuid::Uuid; // by resource. mod address_lot; pub(crate) mod background; +mod bgp; mod certificate; mod device_auth; mod disk; @@ -162,6 +164,9 @@ pub struct Nexus { /// Mapping of SwitchLocations to their respective Dendrite Clients dpd_clients: HashMap>, + /// Map switch location to maghemite admin clients. + mg_clients: HashMap>, + /// Background tasks background_tasks: background::BackgroundTasks, @@ -216,7 +221,13 @@ impl Nexus { let mut dpd_clients: HashMap> = HashMap::new(); - // Currently static dpd configuration mappings are still required for testing + let mut mg_clients: HashMap< + SwitchLocation, + Arc, + > = HashMap::new(); + + // Currently static dpd configuration mappings are still required for + // testing for (location, config) in &config.pkg.dendrite { let address = config.address.ip().to_string(); let port = config.address.port(); @@ -226,6 +237,11 @@ impl Nexus { ); dpd_clients.insert(*location, Arc::new(dpd_client)); } + for (location, config) in &config.pkg.mgd { + let mg_client = mg_admin_client::Client::new(&log, config.address) + .map_err(|e| format!("mg admin client: {e}"))?; + mg_clients.insert(*location, Arc::new(mg_client)); + } if config.pkg.dendrite.is_empty() { loop { let result = resolver @@ -259,6 +275,42 @@ impl Nexus { } } } + if config.pkg.mgd.is_empty() { + loop { + let result = resolver + // TODO this should be ServiceName::Mgd, but in the upgrade + // path, that does not exist because RSS has not + // created it. So we just piggyback on Dendrite's SRV + // record. + .lookup_all_ipv6(ServiceName::Dendrite) + .await + .map_err(|e| format!("Cannot lookup mgd addresses: {e}")); + match result { + Ok(addrs) => { + let mappings = map_switch_zone_addrs( + &log.new(o!("component" => "Nexus")), + addrs, + ) + .await; + for (location, addr) in &mappings { + let port = MGD_PORT; + let mgd_client = mg_admin_client::Client::new( + &log, + std::net::SocketAddr::new((*addr).into(), port), + ) + .map_err(|e| format!("mg admin client: {e}"))?; + mg_clients.insert(*location, Arc::new(mgd_client)); + } + break; + } + Err(e) => { + warn!(log, "Failed to lookup mgd address: {e}"); + tokio::time::sleep(std::time::Duration::from_secs(1)) + .await; + } + } + } + } // Connect to clickhouse - but do so lazily. // Clickhouse may not be executing when Nexus starts. @@ -343,6 +395,7 @@ impl Nexus { .external_dns_servers .clone(), dpd_clients, + mg_clients, background_tasks, default_region_allocation_strategy: config .pkg @@ -352,6 +405,12 @@ impl Nexus { // TODO-cleanup all the extra Arcs here seems wrong let nexus = Arc::new(nexus); + let bootstore_opctx = OpContext::for_background( + log.new(o!("component" => "Bootstore")), + Arc::clone(&authz), + authn::Context::internal_api(), + Arc::clone(&db_datastore), + ); let opctx = OpContext::for_background( log.new(o!("component" => "SagaRecoverer")), Arc::clone(&authz), @@ -391,6 +450,12 @@ impl Nexus { for task in task_nexus.background_tasks.driver.tasks() { task_nexus.background_tasks.driver.activate(task); } + if let Err(e) = task_nexus + .initial_bootstore_sync(&bootstore_opctx) + .await + { + error!(task_log, "failed to run bootstore sync: {e}"); + } } Err(_) => { error!(task_log, "populate failed"); diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 3ac4b9063d..907c3ffa78 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -5,11 +5,14 @@ //! Rack management use super::silo::silo_dns_name; +use crate::external_api::params; use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; +use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -33,17 +36,21 @@ use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::IdentityMetadataCreateParams; -use omicron_common::api::external::IpNet; -use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::Name; use omicron_common::api::external::NameOrId; use omicron_common::api::internal::shared::ExternalPortDiscovery; +use sled_agent_client::types::EarlyNetworkConfigBody; +use sled_agent_client::types::{ + BgpConfig, BgpPeerConfig, EarlyNetworkConfig, PortConfigV1, + RackNetworkConfigV1, RouteConfig as SledRouteConfig, +}; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; +use std::net::Ipv4Addr; use std::str::FromStr; use uuid::Uuid; @@ -188,10 +195,18 @@ impl super::Nexus { mapped_fleet_roles, }; + let rack_network_config = request.rack_network_config.as_ref().ok_or( + Error::InvalidRequest { + message: "cannot initialize a rack without a network config" + .into(), + }, + )?; + self.db_datastore .rack_set_initialized( opctx, RackInit { + rack_subnet: rack_network_config.rack_subnet.into(), rack_id, services: request.services, datasets, @@ -380,7 +395,7 @@ impl super::Nexus { })?; for (idx, uplink_config) in - rack_network_config.uplinks.iter().enumerate() + rack_network_config.ports.iter().enumerate() { let switch = uplink_config.switch.to_string(); let switch_location = Name::from_str(&switch).map_err(|e| { @@ -449,31 +464,32 @@ impl super::Nexus { addresses: HashMap::new(), }; - let uplink_address = - IpNet::V4(Ipv4Net(uplink_config.uplink_cidr)); - let address = Address { - address_lot: NameOrId::Name(address_lot_name.clone()), - address: uplink_address, - }; - port_settings_params.addresses.insert( - "phy0".to_string(), - AddressConfig { addresses: vec![address] }, - ); - - let dst = IpNet::from_str("0.0.0.0/0").map_err(|e| { - Error::internal_error(&format!( - "failed to parse provided default route CIDR: {e}" - )) - })?; - - let gw = IpAddr::V4(uplink_config.gateway_ip); - let vid = uplink_config.uplink_vid; - let route = Route { dst, gw, vid }; - - port_settings_params.routes.insert( - "phy0".to_string(), - RouteConfig { routes: vec![route] }, - ); + let addresses: Vec
= uplink_config + .addresses + .iter() + .map(|a| Address { + address_lot: NameOrId::Name(address_lot_name.clone()), + address: (*a).into(), + }) + .collect(); + + port_settings_params + .addresses + .insert("phy0".to_string(), AddressConfig { addresses }); + + let routes: Vec = uplink_config + .routes + .iter() + .map(|r| Route { + dst: r.destination.into(), + gw: r.nexthop, + vid: None, + }) + .collect(); + + port_settings_params + .routes + .insert("phy0".to_string(), RouteConfig { routes }); match self .db_datastore @@ -498,9 +514,7 @@ impl super::Nexus { opctx, rack_id, switch_location.into(), - Name::from_str(&uplink_config.uplink_port) - .unwrap() - .into(), + Name::from_str(&uplink_config.port).unwrap().into(), ) .await?; @@ -515,6 +529,7 @@ impl super::Nexus { } // TODO - https://github.com/oxidecomputer/omicron/issues/3277 // record port speed }; + self.initial_bootstore_sync(&opctx).await?; Ok(()) } @@ -548,4 +563,153 @@ impl super::Nexus { tokio::time::sleep(std::time::Duration::from_secs(2)).await; } } + + pub(crate) async fn initial_bootstore_sync( + &self, + opctx: &OpContext, + ) -> Result<(), Error> { + let mut rack = self.rack_lookup(opctx, &self.rack_id).await?; + if rack.rack_subnet.is_some() { + return Ok(()); + } + let addr = self + .sled_list(opctx, &DataPageParams::max_page()) + .await? + .get(0) + .ok_or(Error::InternalError { + internal_message: "no sleds at time of bootstore sync".into(), + })? + .address(); + + let sa = sled_agent_client::Client::new( + &format!("http://{}", addr), + self.log.clone(), + ); + + let result = sa + .read_network_bootstore_config_cache() + .await + .map_err(|e| Error::InternalError { + internal_message: format!("read bootstore network config: {e}"), + })? + .into_inner(); + + rack.rack_subnet = + result.body.rack_network_config.map(|x| x.rack_subnet.into()); + + self.datastore().update_rack_subnet(opctx, &rack).await?; + + Ok(()) + } + + pub(crate) async fn bootstore_network_config( + &self, + opctx: &OpContext, + ) -> Result { + let rack = self.rack_lookup(opctx, &self.rack_id).await?; + + let subnet = match rack.rack_subnet { + Some(IpNetwork::V6(subnet)) => subnet, + Some(IpNetwork::V4(_)) => { + return Err(Error::InternalError { + internal_message: "rack subnet not IPv6".into(), + }) + } + None => { + return Err(Error::InternalError { + internal_message: "rack subnet not set".into(), + }) + } + }; + + let db_ports = self.active_port_settings(opctx).await?; + let mut ports = Vec::new(); + let mut bgp = Vec::new(); + for (port, info) in &db_ports { + let mut peer_info = Vec::new(); + for p in &info.bgp_peers { + let bgp_config = + self.bgp_config_get(&opctx, p.bgp_config_id.into()).await?; + let announcements = self + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: bgp_config.bgp_announce_set_id.into(), + }, + ) + .await?; + let addr = match p.addr { + ipnetwork::IpNetwork::V4(addr) => addr, + ipnetwork::IpNetwork::V6(_) => continue, //TODO v6 + }; + peer_info.push((p, bgp_config.asn.0, addr.ip())); + bgp.push(BgpConfig { + asn: bgp_config.asn.0, + originate: announcements + .iter() + .filter_map(|a| match a.network { + IpNetwork::V4(net) => Some(net.into()), + //TODO v6 + _ => None, + }) + .collect(), + }); + } + + let p = PortConfigV1 { + routes: info + .routes + .iter() + .map(|r| SledRouteConfig { + destination: r.dst, + nexthop: r.gw.ip(), + }) + .collect(), + addresses: info.addresses.iter().map(|a| a.address).collect(), + bgp_peers: peer_info + .iter() + .map(|(_p, asn, addr)| BgpPeerConfig { + addr: *addr, + asn: *asn, + port: port.port_name.clone(), + }) + .collect(), + switch: port.switch_location.parse().unwrap(), + port: port.port_name.clone(), + uplink_port_fec: info + .links + .get(0) //TODO breakout support + .map(|l| l.fec) + .unwrap_or(SwitchLinkFec::None) + .into(), + uplink_port_speed: info + .links + .get(0) //TODO breakout support + .map(|l| l.speed) + .unwrap_or(SwitchLinkSpeed::Speed100G) + .into(), + }; + + ports.push(p); + } + + let result = EarlyNetworkConfig { + generation: 0, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: Vec::new(), //TODO + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: subnet, + //TODO(ry) you are here. We need to remove these too. They are + // inconsistent with a generic set of addresses on ports. + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports, + bgp, + }), + }, + }; + + Ok(result) + } } diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs index 0ae17c7237..83e0e9b8b4 100644 --- a/nexus/src/app/sagas/mod.rs +++ b/nexus/src/app/sagas/mod.rs @@ -36,7 +36,6 @@ pub mod snapshot_create; pub mod snapshot_delete; pub mod switch_port_settings_apply; pub mod switch_port_settings_clear; -pub mod switch_port_settings_update; pub mod test_saga; pub mod volume_delete; pub mod volume_remove_rop; diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index 687613f0cc..93dc45751a 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -7,6 +7,7 @@ use crate::app::sagas::retry_until_known_result; use crate::app::sagas::{ declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError, }; +use crate::Nexus; use anyhow::Error; use db::datastore::SwitchPortSettingsCombinedResult; use dpd_client::types::{ @@ -15,18 +16,37 @@ use dpd_client::types::{ }; use dpd_client::{Ipv4Cidr, Ipv6Cidr}; use ipnetwork::IpNetwork; +use mg_admin_client::types::Prefix4; +use mg_admin_client::types::{ApplyRequest, BgpPeerConfig, BgpRoute}; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed, NETWORK_KEY}; +use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::UpdatePrecondition; use nexus_db_queries::{authn, db}; -use omicron_common::api::external::{self, NameOrId}; -use omicron_common::api::internal::shared::SwitchLocation; +use nexus_types::external_api::params; +use omicron_common::api::external::{self, DataPageParams, NameOrId}; +use omicron_common::api::internal::shared::{ + ParseSwitchLocationError, SwitchLocation, +}; use serde::{Deserialize, Serialize}; +use sled_agent_client::types::PortConfigV1; +use sled_agent_client::types::RouteConfig; +use sled_agent_client::types::{BgpConfig, EarlyNetworkConfig}; +use sled_agent_client::types::{ + BgpPeerConfig as OmicronBgpPeerConfig, HostPortConfig, +}; use std::collections::HashMap; use std::net::IpAddr; +use std::net::SocketAddrV6; use std::str::FromStr; use std::sync::Arc; use steno::ActionError; use uuid::Uuid; +// This is more of an implementation detail of the BGP implementation. It +// defines the maximum time the peering engine will wait for external messages +// before breaking to check for shutdown conditions. +const BGP_SESSION_RESOLUTION: u64 = 100; + // switch port settings apply saga: input parameters #[derive(Debug, Deserialize, Serialize)] @@ -52,6 +72,18 @@ declare_saga_actions! { + spa_ensure_switch_port_settings - spa_undo_ensure_switch_port_settings } + ENSURE_SWITCH_PORT_UPLINK -> "ensure_switch_port_uplink" { + + spa_ensure_switch_port_uplink + - spa_undo_ensure_switch_port_uplink + } + ENSURE_SWITCH_PORT_BGP_SETTINGS -> "ensure_switch_port_bgp_settings" { + + spa_ensure_switch_port_bgp_settings + - spa_undo_ensure_switch_port_bgp_settings + } + ENSURE_SWITCH_PORT_BOOTSTORE_NETWORK_SETTINGS -> "ensure_switch_port_bootstore_network_settings" { + + spa_ensure_switch_port_bootstore_network_settings + - spa_undo_ensure_switch_port_bootstore_network_settings + } } // switch port settings apply saga: definition @@ -74,6 +106,9 @@ impl NexusSaga for SagaSwitchPortSettingsApply { builder.append(associate_switch_port_action()); builder.append(get_switch_port_settings_action()); builder.append(ensure_switch_port_settings_action()); + builder.append(ensure_switch_port_uplink_action()); + builder.append(ensure_switch_port_bgp_settings_action()); + builder.append(ensure_switch_port_bootstore_network_settings_action()); Ok(builder.build()?) } } @@ -91,10 +126,10 @@ async fn spa_associate_switch_port( ); // first get the current association so we fall back to this on failure - let port = nexus - .get_switch_port(&opctx, params.switch_port_id) - .await - .map_err(ActionError::action_failed)?; + let port = + nexus.get_switch_port(&opctx, params.switch_port_id).await.map_err( + |e| ActionError::action_failed(format!("get switch port: {e}")), + )?; // update the switch port settings association nexus @@ -105,7 +140,11 @@ async fn spa_associate_switch_port( UpdatePrecondition::DontCare, ) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!( + "set switch port settings id {e}" + )) + })?; Ok(port.port_settings_id) } @@ -127,7 +166,9 @@ async fn spa_get_switch_port_settings( &NameOrId::Id(params.switch_port_settings_id), ) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!("get switch port settings: {e}")) + })?; Ok(port_settings) } @@ -142,22 +183,42 @@ pub(crate) fn api_to_dpd_port_settings( v6_routes: HashMap::new(), }; - // TODO handle breakouts - // https://github.com/oxidecomputer/omicron/issues/3062 + //TODO breakouts let link_id = LinkId(0); - let link_settings = LinkSettings { - // TODO Allow user to configure link properties - // https://github.com/oxidecomputer/omicron/issues/3061 - params: LinkCreate { - autoneg: false, - kr: false, - fec: PortFec::None, - speed: PortSpeed::Speed100G, - }, - addrs: settings.addresses.iter().map(|a| a.address.ip()).collect(), - }; - dpd_port_settings.links.insert(link_id.to_string(), link_settings); + for l in settings.links.iter() { + dpd_port_settings.links.insert( + link_id.to_string(), + LinkSettings { + params: LinkCreate { + autoneg: false, + kr: false, + fec: match l.fec { + SwitchLinkFec::Firecode => PortFec::Firecode, + SwitchLinkFec::Rs => PortFec::Rs, + SwitchLinkFec::None => PortFec::None, + }, + speed: match l.speed { + SwitchLinkSpeed::Speed0G => PortSpeed::Speed0G, + SwitchLinkSpeed::Speed1G => PortSpeed::Speed1G, + SwitchLinkSpeed::Speed10G => PortSpeed::Speed10G, + SwitchLinkSpeed::Speed25G => PortSpeed::Speed25G, + SwitchLinkSpeed::Speed40G => PortSpeed::Speed40G, + SwitchLinkSpeed::Speed50G => PortSpeed::Speed50G, + SwitchLinkSpeed::Speed100G => PortSpeed::Speed100G, + SwitchLinkSpeed::Speed200G => PortSpeed::Speed200G, + SwitchLinkSpeed::Speed400G => PortSpeed::Speed400G, + }, + }, + //TODO won't work for breakouts + addrs: settings + .addresses + .iter() + .map(|a| a.address.ip()) + .collect(), + }, + ); + } for r in &settings.routes { match &r.dst { @@ -214,20 +275,28 @@ async fn spa_ensure_switch_port_settings( let settings = sagactx .lookup::("switch_port_settings")?; - let port_id: PortId = PortId::from_str(¶ms.switch_port_name) - .map_err(|e| ActionError::action_failed(e.to_string()))?; + let port_id: PortId = + PortId::from_str(¶ms.switch_port_name).map_err(|e| { + ActionError::action_failed(format!("parse port id: {e}")) + })?; let dpd_client: Arc = select_dendrite_client(&sagactx).await?; - let dpd_port_settings = api_to_dpd_port_settings(&settings) - .map_err(ActionError::action_failed)?; + let dpd_port_settings = + api_to_dpd_port_settings(&settings).map_err(|e| { + ActionError::action_failed(format!( + "translate api port settings to dpd port settings: {e}", + )) + })?; retry_until_known_result(log, || async { dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await }) .await - .map_err(|e| ActionError::action_failed(e.to_string()))?; + .map_err(|e| { + ActionError::action_failed(format!("dpd port settings apply {e}")) + })?; Ok(()) } @@ -270,10 +339,16 @@ async fn spa_undo_ensure_switch_port_settings( let settings = nexus .switch_port_settings_get(&opctx, &NameOrId::Id(id)) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!("switch port settings get: {e}")) + })?; - let dpd_port_settings = api_to_dpd_port_settings(&settings) - .map_err(ActionError::action_failed)?; + let dpd_port_settings = + api_to_dpd_port_settings(&settings).map_err(|e| { + ActionError::action_failed(format!( + "translate api to dpd port settings {e}" + )) + })?; retry_until_known_result(log, || async { dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await @@ -284,6 +359,326 @@ async fn spa_undo_ensure_switch_port_settings( Ok(()) } +async fn spa_ensure_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let settings = sagactx + .lookup::("switch_port_settings") + .map_err(|e| { + ActionError::action_failed(format!( + "lookup switch port settings: {e}" + )) + })?; + + ensure_switch_port_bgp_settings(sagactx, settings).await +} + +pub(crate) async fn ensure_switch_port_bgp_settings( + sagactx: NexusActionContext, + settings: SwitchPortSettingsCombinedResult, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let mg_client: Arc = + select_mg_client(&sagactx).await.map_err(|e| { + ActionError::action_failed(format!("select mg client: {e}")) + })?; + + let mut bgp_peer_configs = Vec::new(); + + for peer in settings.bgp_peers { + let config = nexus + .bgp_config_get(&opctx, peer.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("get bgp config: {e}")) + })?; + + let announcements = nexus + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: NameOrId::Id(config.bgp_announce_set_id), + }, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get bgp announcements: {e}" + )) + })?; + + // TODO picking the first configured address by default, but this needs + // to be something that can be specified in the API. + let nexthop = match settings.addresses.get(0) { + Some(switch_port_addr) => Ok(switch_port_addr.address.ip()), + None => Err(ActionError::action_failed( + "at least one address required for bgp peering".to_string(), + )), + }?; + + let nexthop = match nexthop { + IpAddr::V4(nexthop) => Ok(nexthop), + IpAddr::V6(_) => Err(ActionError::action_failed( + "IPv6 nexthop not yet supported".to_string(), + )), + }?; + + let mut prefixes = Vec::new(); + for a in &announcements { + let value = match a.network.ip() { + IpAddr::V4(value) => Ok(value), + IpAddr::V6(_) => Err(ActionError::action_failed( + "IPv6 announcement not yet supported".to_string(), + )), + }?; + prefixes.push(Prefix4 { value, length: a.network.prefix() }); + } + + let bpc = BgpPeerConfig { + asn: *config.asn, + name: format!("{}", peer.addr.ip()), //TODO user defined name? + host: format!("{}:179", peer.addr.ip()), + hold_time: peer.hold_time.0.into(), + idle_hold_time: peer.idle_hold_time.0.into(), + delay_open: peer.delay_open.0.into(), + connect_retry: peer.connect_retry.0.into(), + keepalive: peer.keepalive.0.into(), + resolution: BGP_SESSION_RESOLUTION, + routes: vec![BgpRoute { nexthop, prefixes }], + }; + + bgp_peer_configs.push(bpc); + } + + mg_client + .inner + .bgp_apply(&ApplyRequest { + peer_group: params.switch_port_name.clone(), + peers: bgp_peer_configs, + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("apply bgp settings: {e}")) + })?; + + Ok(()) +} +async fn spa_undo_ensure_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + use mg_admin_client::types::DeleteNeighborRequest; + + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let settings = sagactx + .lookup::("switch_port_settings") + .map_err(|e| { + ActionError::action_failed(format!( + "lookup switch port settings (bgp undo): {e}" + )) + })?; + + let mg_client: Arc = + select_mg_client(&sagactx).await.map_err(|e| { + ActionError::action_failed(format!("select mg client (undo): {e}")) + })?; + + for peer in settings.bgp_peers { + let config = nexus + .bgp_config_get(&opctx, peer.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete bgp config: {e}")) + })?; + + mg_client + .inner + .delete_neighbor(&DeleteNeighborRequest { + asn: *config.asn, + addr: peer.addr.ip(), + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete neighbor: {e}")) + })?; + } + + Ok(()) +} + +async fn spa_ensure_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + let mut config = + nexus.bootstore_network_config(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "read nexus bootstore network config: {e}" + )) + })?; + + let generation = nexus + .datastore() + .bump_bootstore_generation(&opctx, NETWORK_KEY.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "bump bootstore network generation number: {e}" + )) + })?; + + config.generation = generation as u64; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} + +async fn spa_undo_ensure_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + // The overall saga update failed but the bootstore udpate succeeded. + // Between now and then other updates may have happened which prevent us + // from simply undoing the changes we did before, as we may inadvertently + // roll back changes at the intersection of this failed update and other + // succesful updates. The only thing we can really do here is attempt a + // complete update of the bootstore network settings based on the current + // state in the Nexus databse which, we assume to be consistent at any point + // in time. + + let nexus = sagactx.user_data().nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + let config = nexus.bootstore_network_config(&opctx).await?; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} + +async fn spa_ensure_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + ensure_switch_port_uplink(sagactx, false, None).await +} + +async fn spa_undo_ensure_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), Error> { + Ok(ensure_switch_port_uplink(sagactx, true, None).await?) +} + +pub(crate) async fn ensure_switch_port_uplink( + sagactx: NexusActionContext, + skip_self: bool, + inject: Option, +) -> Result<(), ActionError> { + let params = sagactx.saga_params::()?; + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + + let switch_port = nexus + .get_switch_port(&opctx, params.switch_port_id) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for uplink: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err(|e| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + })?; + + let mut uplinks: Vec = Vec::new(); + + // The sled agent uplinks interface is an all or nothing interface, so we + // need to get all the uplink configs for all the ports. + let active_ports = + nexus.active_port_settings(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "get active switch port settings: {e}" + )) + })?; + + for (port, info) in &active_ports { + // Since we are undoing establishing uplinks for the settings + // associated with this port we skip adding this ports uplinks + // to the list - effectively removing them. + if skip_self && port.id == switch_port.id { + continue; + } + uplinks.push(HostPortConfig { + port: port.port_name.clone(), + addrs: info.addresses.iter().map(|a| a.address).collect(), + }) + } + + if let Some(id) = inject { + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let settings = nexus + .switch_port_settings_get(&opctx, &id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port settings for injection: {e}" + )) + })?; + uplinks.push(HostPortConfig { + port: params.switch_port_name.clone(), + addrs: settings.addresses.iter().map(|a| a.address).collect(), + }) + } + + let sc = switch_sled_agent(switch_location, &sagactx).await?; + sc.uplink_ensure(&sled_agent_client::types::SwitchPorts { uplinks }) + .await + .map_err(|e| { + ActionError::action_failed(format!("ensure uplink: {e}")) + })?; + + Ok(()) +} + // a common route representation for dendrite and port settings #[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)] pub(crate) struct Route { @@ -316,7 +711,11 @@ async fn spa_disassociate_switch_port( UpdatePrecondition::Value(params.switch_port_settings_id), ) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!( + "set switch port settings id for disassociate: {e}" + )) + })?; Ok(()) } @@ -335,12 +734,21 @@ pub(crate) async fn select_dendrite_client( let switch_port = nexus .get_switch_port(&opctx, params.switch_port_id) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for dendrite client selection {e}" + )) + })?; + let switch_location: SwitchLocation = - switch_port - .switch_location - .parse() - .map_err(ActionError::action_failed)?; + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + let dpd_client: Arc = osagactx .nexus() .dpd_clients @@ -353,3 +761,283 @@ pub(crate) async fn select_dendrite_client( .clone(); Ok(dpd_client) } + +pub(crate) async fn select_mg_client( + sagactx: &NexusActionContext, +) -> Result, ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let switch_port = nexus + .get_switch_port(&opctx, params.switch_port_id) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for mg client selection: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + + let mg_client: Arc = osagactx + .nexus() + .mg_clients + .get(&switch_location) + .ok_or_else(|| { + ActionError::action_failed(format!( + "requested switch not available: {switch_location}" + )) + })? + .clone(); + Ok(mg_client) +} + +pub(crate) async fn get_scrimlet_address( + _location: SwitchLocation, + nexus: &Arc, +) -> Result { + /* TODO this depends on DNS entries only coming from RSS, it's broken + on the upgrade path + nexus + .resolver() + .await + .lookup_socket_v6(ServiceName::Scrimlet(location)) + .await + .map_err(|e| e.to_string()) + .map_err(|e| { + ActionError::action_failed(format!( + "scrimlet dns lookup failed {e}", + )) + }) + */ + let opctx = &nexus.opctx_for_internal_api(); + Ok(nexus + .sled_list(opctx, &DataPageParams::max_page()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get_scrimlet_address: failed to list sleds: {e}" + )) + })? + .into_iter() + .find(|x| x.is_scrimlet()) + .ok_or(ActionError::action_failed( + "get_scrimlet_address: no scrimlets found".to_string(), + ))? + .address()) +} + +#[derive(Clone, Debug)] +pub struct EarlyNetworkPortUpdate { + port: PortConfigV1, + bgp_configs: Vec, +} + +pub(crate) async fn bootstore_update( + nexus: &Arc, + opctx: &OpContext, + switch_port_id: Uuid, + switch_port_name: &str, + settings: &SwitchPortSettingsCombinedResult, +) -> Result { + let switch_port = + nexus.get_switch_port(&opctx, switch_port_id).await.map_err(|e| { + ActionError::action_failed(format!( + "get switch port for uplink: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + + let mut peer_info = Vec::new(); + let mut bgp_configs = Vec::new(); + for p in &settings.bgp_peers { + let bgp_config = nexus + .bgp_config_get(&opctx, p.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("get bgp config: {e}")) + })?; + + let announcements = nexus + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: NameOrId::Id(bgp_config.bgp_announce_set_id), + }, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get bgp announcements: {e}" + )) + })?; + + peer_info.push((p, bgp_config.asn.0)); + bgp_configs.push(BgpConfig { + asn: bgp_config.asn.0, + originate: announcements + .iter() + .filter_map(|a| match a.network { + IpNetwork::V4(net) => Some(net.into()), + //TODO v6 + _ => None, + }) + .collect(), + }); + } + + let update = EarlyNetworkPortUpdate { + port: PortConfigV1 { + routes: settings + .routes + .iter() + .map(|r| RouteConfig { destination: r.dst, nexthop: r.gw.ip() }) + .collect(), + addresses: settings.addresses.iter().map(|a| a.address).collect(), + switch: switch_location, + port: switch_port_name.into(), + uplink_port_fec: settings + .links + .get(0) + .map(|l| l.fec) + .unwrap_or(SwitchLinkFec::None) + .into(), + uplink_port_speed: settings + .links + .get(0) + .map(|l| l.speed) + .unwrap_or(SwitchLinkSpeed::Speed100G) + .into(), + bgp_peers: peer_info + .iter() + .filter_map(|(p, asn)| { + //TODO v6 + match p.addr.ip() { + IpAddr::V4(addr) => Some(OmicronBgpPeerConfig { + asn: *asn, + port: switch_port_name.into(), + addr, + }), + IpAddr::V6(_) => { + warn!(opctx.log, "IPv6 peers not yet supported"); + None + } + } + }) + .collect(), + }, + bgp_configs, + }; + + Ok(update) +} + +pub(crate) async fn read_bootstore_config( + sa: &sled_agent_client::Client, +) -> Result { + Ok(sa + .read_network_bootstore_config_cache() + .await + .map_err(|e| { + ActionError::action_failed(format!( + "read bootstore network config: {e}" + )) + })? + .into_inner()) +} + +pub(crate) async fn write_bootstore_config( + sa: &sled_agent_client::Client, + config: &EarlyNetworkConfig, +) -> Result<(), ActionError> { + sa.write_network_bootstore_config(config).await.map_err(|e| { + ActionError::action_failed(format!( + "write bootstore network config: {e}" + )) + })?; + Ok(()) +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct BootstoreNetworkPortChange { + previous_port_config: Option, + changed_bgp_configs: Vec, + added_bgp_configs: Vec, +} + +pub(crate) fn apply_bootstore_update( + config: &mut EarlyNetworkConfig, + update: &EarlyNetworkPortUpdate, +) -> Result { + let mut change = BootstoreNetworkPortChange::default(); + + let rack_net_config = match &mut config.body.rack_network_config { + Some(cfg) => cfg, + None => { + return Err(ActionError::action_failed( + "rack network config not yet initialized".to_string(), + )) + } + }; + + for port in &mut rack_net_config.ports { + if port.port == update.port.port { + change.previous_port_config = Some(port.clone()); + *port = update.port.clone(); + break; + } + } + if change.previous_port_config.is_none() { + rack_net_config.ports.push(update.port.clone()); + } + + for updated_bgp in &update.bgp_configs { + let mut exists = false; + for resident_bgp in &mut rack_net_config.bgp { + if resident_bgp.asn == updated_bgp.asn { + change.changed_bgp_configs.push(resident_bgp.clone()); + *resident_bgp = updated_bgp.clone(); + exists = true; + break; + } + } + if !exists { + change.added_bgp_configs.push(updated_bgp.clone()); + } + } + rack_net_config.bgp.extend_from_slice(&change.added_bgp_configs); + + Ok(change) +} + +pub(crate) async fn switch_sled_agent( + location: SwitchLocation, + sagactx: &NexusActionContext, +) -> Result { + let nexus = sagactx.user_data().nexus(); + let sled_agent_addr = get_scrimlet_address(location, nexus).await?; + Ok(sled_agent_client::Client::new( + &format!("http://{}", sled_agent_addr), + sagactx.user_data().log().clone(), + )) +} diff --git a/nexus/src/app/sagas/switch_port_settings_clear.rs b/nexus/src/app/sagas/switch_port_settings_clear.rs index 0c0f4ec01b..14544b0f55 100644 --- a/nexus/src/app/sagas/switch_port_settings_clear.rs +++ b/nexus/src/app/sagas/switch_port_settings_clear.rs @@ -5,17 +5,25 @@ use super::switch_port_settings_apply::select_dendrite_client; use super::NexusActionContext; use crate::app::sagas::retry_until_known_result; -use crate::app::sagas::switch_port_settings_apply::api_to_dpd_port_settings; +use crate::app::sagas::switch_port_settings_apply::{ + api_to_dpd_port_settings, apply_bootstore_update, bootstore_update, + ensure_switch_port_bgp_settings, ensure_switch_port_uplink, + read_bootstore_config, select_mg_client, switch_sled_agent, + write_bootstore_config, +}; use crate::app::sagas::{ declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError, }; use anyhow::Error; use dpd_client::types::PortId; +use mg_admin_client::types::DeleteNeighborRequest; +use nexus_db_model::NETWORK_KEY; use nexus_db_queries::authn; use nexus_db_queries::db::datastore::UpdatePrecondition; -use omicron_common::api::external::{self, NameOrId}; +use omicron_common::api::external::{self, NameOrId, SwitchLocation}; use serde::{Deserialize, Serialize}; use std::str::FromStr; +use std::sync::Arc; use steno::ActionError; use uuid::Uuid; @@ -36,6 +44,18 @@ declare_saga_actions! { + spa_clear_switch_port_settings - spa_undo_clear_switch_port_settings } + CLEAR_SWITCH_PORT_UPLINK -> "clear_switch_port_uplink" { + + spa_clear_switch_port_uplink + - spa_undo_clear_switch_port_uplink + } + CLEAR_SWITCH_PORT_BGP_SETTINGS -> "clear_switch_port_bgp_settings" { + + spa_clear_switch_port_bgp_settings + - spa_undo_clear_switch_port_bgp_settings + } + CLEAR_SWITCH_PORT_BOOTSTORE_NETWORK_SETTINGS -> "clear_switch_port_bootstore_network_settings" { + + spa_clear_switch_port_bootstore_network_settings + - spa_undo_clear_switch_port_bootstore_network_settings + } } #[derive(Debug)] @@ -54,6 +74,9 @@ impl NexusSaga for SagaSwitchPortSettingsClear { ) -> Result { builder.append(disassociate_switch_port_action()); builder.append(clear_switch_port_settings_action()); + builder.append(clear_switch_port_uplink_action()); + builder.append(clear_switch_port_bgp_settings_action()); + builder.append(clear_switch_port_bootstore_network_settings_action()); Ok(builder.build()?) } } @@ -181,3 +204,185 @@ async fn spa_undo_clear_switch_port_settings( Ok(()) } + +async fn spa_clear_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + ensure_switch_port_uplink(sagactx, true, None).await +} + +async fn spa_undo_clear_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let id = sagactx + .lookup::>("original_switch_port_settings_id") + .map_err(|e| external::Error::internal_error(&e.to_string()))?; + + Ok(ensure_switch_port_uplink(sagactx, false, id).await?) +} + +async fn spa_clear_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = sagactx + .lookup::>("original_switch_port_settings_id") + .map_err(|e| { + ActionError::action_failed(format!( + "original port settings id lookup: {e}" + )) + })?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = nexus + .switch_port_settings_get(&opctx, &NameOrId::Id(id)) + .await + .map_err(ActionError::action_failed)?; + + let mg_client: Arc = + select_mg_client(&sagactx).await.map_err(|e| { + ActionError::action_failed(format!("select mg client (undo): {e}")) + })?; + + for peer in settings.bgp_peers { + let config = nexus + .bgp_config_get(&opctx, peer.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete bgp config: {e}")) + })?; + + mg_client + .inner + .delete_neighbor(&DeleteNeighborRequest { + asn: *config.asn, + addr: peer.addr.ip(), + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete neighbor: {e}")) + })?; + } + + Ok(()) +} + +async fn spa_undo_clear_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = + sagactx.lookup::>("original_switch_port_settings_id")?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = + nexus.switch_port_settings_get(&opctx, &NameOrId::Id(id)).await?; + + Ok(ensure_switch_port_bgp_settings(sagactx, settings).await?) +} + +async fn spa_clear_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let nexus = sagactx.user_data().nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + let mut config = + nexus.bootstore_network_config(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "read nexus bootstore network config: {e}" + )) + })?; + + let generation = nexus + .datastore() + .bump_bootstore_generation(&opctx, NETWORK_KEY.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "bump bootstore network generation number: {e}" + )) + })?; + + config.generation = generation as u64; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} + +async fn spa_undo_clear_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = sagactx + .lookup::>("original_switch_port_settings_id") + .map_err(|e| { + ActionError::action_failed(format!( + "original port settings id lookup: {e}" + )) + })?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = nexus + .switch_port_settings_get(&opctx, &NameOrId::Id(id)) + .await + .map_err(ActionError::action_failed)?; + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + // Read the current bootstore config, perform the update and write it back. + let mut config = read_bootstore_config(&sa).await?; + let update = bootstore_update( + &nexus, + &opctx, + params.switch_port_id, + ¶ms.port_name, + &settings, + ) + .await?; + apply_bootstore_update(&mut config, &update)?; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} diff --git a/nexus/src/app/sagas/switch_port_settings_update.rs b/nexus/src/app/sagas/switch_port_settings_update.rs deleted file mode 100644 index 23120bdbf4..0000000000 --- a/nexus/src/app/sagas/switch_port_settings_update.rs +++ /dev/null @@ -1,5 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// TODO https://github.com/oxidecomputer/omicron/issues/3002 diff --git a/nexus/src/app/switch_port.rs b/nexus/src/app/switch_port.rs index 996290b684..03b874727b 100644 --- a/nexus/src/app/switch_port.rs +++ b/nexus/src/app/switch_port.rs @@ -2,33 +2,114 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//XXX +#![allow(unused_imports)] + use crate::app::sagas; use crate::external_api::params; use db::datastore::SwitchPortSettingsCombinedResult; +use ipnetwork::IpNetwork; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::authn; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use nexus_db_queries::db::datastore::UpdatePrecondition; use nexus_db_queries::db::model::{SwitchPort, SwitchPortSettings}; +use nexus_types::identity::Resource; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::{ self, CreateResult, DataPageParams, DeleteResult, ListResultVec, LookupResult, Name, NameOrId, UpdateResult, }; +use sled_agent_client::types::BgpConfig; +use sled_agent_client::types::BgpPeerConfig; +use sled_agent_client::types::{ + EarlyNetworkConfig, PortConfigV1, RackNetworkConfigV1, RouteConfig, +}; use std::sync::Arc; use uuid::Uuid; impl super::Nexus { - pub(crate) async fn switch_port_settings_create( - &self, + pub(crate) async fn switch_port_settings_post( + self: &Arc, opctx: &OpContext, params: params::SwitchPortSettingsCreate, ) -> CreateResult { opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + + //TODO(ry) race conditions on exists check versus update/create. + // Normally I would use a DB lock here, but not sure what + // the Omicron way of doing things here is. + + match self + .db_datastore + .switch_port_settings_exist( + opctx, + params.identity.name.clone().into(), + ) + .await + { + Ok(id) => self.switch_port_settings_update(opctx, id, params).await, + Err(_) => self.switch_port_settings_create(opctx, params).await, + } + } + + pub async fn switch_port_settings_create( + self: &Arc, + opctx: &OpContext, + params: params::SwitchPortSettingsCreate, + ) -> CreateResult { self.db_datastore.switch_port_settings_create(opctx, ¶ms).await } + pub(crate) async fn switch_port_settings_update( + self: &Arc, + opctx: &OpContext, + switch_port_settings_id: Uuid, + new_settings: params::SwitchPortSettingsCreate, + ) -> CreateResult { + // delete old settings + self.switch_port_settings_delete( + opctx, + ¶ms::SwitchPortSettingsSelector { + port_settings: Some(NameOrId::Id(switch_port_settings_id)), + }, + ) + .await?; + + // create new settings + let result = self + .switch_port_settings_create(opctx, new_settings.clone()) + .await?; + + // run the port settings apply saga for each port referencing the + // updated settings + + let ports = self + .db_datastore + .switch_ports_using_settings(opctx, switch_port_settings_id) + .await?; + + for (switch_port_id, switch_port_name) in ports.into_iter() { + let saga_params = sagas::switch_port_settings_apply::Params { + serialized_authn: authn::saga::Serialized::for_opctx(opctx), + switch_port_id, + switch_port_settings_id: result.settings.id(), + switch_port_name: switch_port_name.to_string(), + }; + + self.execute_saga::< + sagas::switch_port_settings_apply::SagaSwitchPortSettingsApply + >( + saga_params, + ) + .await?; + } + + Ok(result) + } + pub(crate) async fn switch_port_settings_delete( &self, opctx: &OpContext, @@ -151,7 +232,9 @@ impl super::Nexus { switch_port_name: port.to_string(), }; - self.execute_saga::( + self.execute_saga::< + sagas::switch_port_settings_apply::SagaSwitchPortSettingsApply + >( saga_params, ) .await?; @@ -215,4 +298,25 @@ impl super::Nexus { Ok(()) } + + // TODO it would likely be better to do this as a one shot db query. + pub(crate) async fn active_port_settings( + &self, + opctx: &OpContext, + ) -> LookupResult> { + let mut ports = Vec::new(); + let port_list = + self.switch_port_list(opctx, &DataPageParams::max_page()).await?; + + for p in port_list { + if let Some(id) = p.port_settings_id { + ports.push(( + p.clone(), + self.switch_port_settings_get(opctx, &id.into()).await?, + )); + } + } + + LookupResult::Ok(ports) + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 1fddfba85b..990704904a 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -63,6 +63,11 @@ use omicron_common::api::external::http_pagination::ScanParams; use omicron_common::api::external::AddressLot; use omicron_common::api::external::AddressLotBlock; use omicron_common::api::external::AddressLotCreateResponse; +use omicron_common::api::external::BgpAnnounceSet; +use omicron_common::api::external::BgpAnnouncement; +use omicron_common::api::external::BgpConfig; +use omicron_common::api::external::BgpImportedRouteIpv4; +use omicron_common::api::external::BgpPeerStatus; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Disk; use omicron_common::api::external::Error; @@ -250,6 +255,15 @@ pub(crate) fn external_api() -> NexusApiDescription { api.register(networking_switch_port_apply_settings)?; api.register(networking_switch_port_clear_settings)?; + api.register(networking_bgp_config_create)?; + api.register(networking_bgp_config_list)?; + api.register(networking_bgp_status)?; + api.register(networking_bgp_imported_routes_ipv4)?; + api.register(networking_bgp_config_delete)?; + api.register(networking_bgp_announce_set_create)?; + api.register(networking_bgp_announce_set_list)?; + api.register(networking_bgp_announce_set_delete)?; + // Fleet-wide API operations api.register(silo_list)?; api.register(silo_create)?; @@ -2642,7 +2656,7 @@ async fn networking_switch_port_settings_create( let nexus = &apictx.nexus; let params = new_settings.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus.switch_port_settings_create(&opctx, params).await?; + let result = nexus.switch_port_settings_post(&opctx, params).await?; let settings: SwitchPortSettingsView = result.into(); Ok(HttpResponseCreated(settings)) @@ -2810,6 +2824,193 @@ async fn networking_switch_port_clear_settings( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } +/// Create a new BGP configuration. +#[endpoint { + method = POST, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], +}] +async fn networking_bgp_config_create( + rqctx: RequestContext>, + config: TypedBody, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let config = config.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.bgp_config_set(&opctx, &config).await?; + Ok(HttpResponseCreated::(result.into())) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Get BGP configurations. +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], +}] +async fn networking_bgp_config_list( + rqctx: RequestContext>, + query_params: Query>, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let configs = nexus + .bgp_config_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + configs, + &marker_for_name_or_id, + )?)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +//TODO pagination? the normal by-name/by-id stuff does not work here +/// Get BGP peer status +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp-status", + tags = ["system/networking"], +}] +async fn networking_bgp_status( + rqctx: RequestContext>, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let handler = async { + let nexus = &apictx.nexus; + let result = nexus.bgp_peer_status(&opctx).await?; + Ok(HttpResponseOk(result)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +//TODO pagination? the normal by-name/by-id stuff does not work here +/// Get imported IPv4 BGP routes. +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp-routes-ipv4", + tags = ["system/networking"], +}] +async fn networking_bgp_imported_routes_ipv4( + rqctx: RequestContext>, + query_params: Query, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let handler = async { + let nexus = &apictx.nexus; + let sel = query_params.into_inner(); + let result = nexus.bgp_imported_routes_ipv4(&opctx, &sel).await?; + Ok(HttpResponseOk(result)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Delete a BGP configuration. +#[endpoint { + method = DELETE, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], +}] +async fn networking_bgp_config_delete( + rqctx: RequestContext>, + sel: Query, +) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let sel = sel.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + nexus.bgp_config_delete(&opctx, &sel).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Create a new BGP announce set. +#[endpoint { + method = POST, + path = "/v1/system/networking/bgp-announce", + tags = ["system/networking"], +}] +async fn networking_bgp_announce_set_create( + rqctx: RequestContext>, + config: TypedBody, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let config = config.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.bgp_create_announce_set(&opctx, &config).await?; + Ok(HttpResponseCreated::(result.0.into())) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +//TODO pagination? the normal by-name/by-id stuff does not work here +/// Get originated routes for a given BGP configuration. +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp-announce", + tags = ["system/networking"], +}] +async fn networking_bgp_announce_set_list( + rqctx: RequestContext>, + query_params: Query, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let sel = query_params.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus + .bgp_announce_list(&opctx, &sel) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + Ok(HttpResponseOk(result)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Delete a BGP announce set. +#[endpoint { + method = DELETE, + path = "/v1/system/networking/bgp-announce", + tags = ["system/networking"], +}] +async fn networking_bgp_announce_set_delete( + rqctx: RequestContext>, + selector: Query, +) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let sel = selector.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + nexus.bgp_delete_announce_set(&opctx, &sel).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + // Images /// List images diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index 0ada48e203..01aca36e1d 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -32,12 +32,12 @@ use internal_api::http_entrypoints::internal_api; use nexus_types::internal_api::params::ServiceKind; use omicron_common::address::IpRange; use omicron_common::api::internal::shared::{ - ExternalPortDiscovery, SwitchLocation, + ExternalPortDiscovery, RackNetworkConfig, SwitchLocation, }; use omicron_common::FileKv; use slog::Logger; use std::collections::HashMap; -use std::net::{SocketAddr, SocketAddrV6}; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -289,7 +289,13 @@ impl nexus_test_interface::NexusServer for Server { vec!["qsfp0".parse().unwrap()], )]), ), - rack_network_config: None, + rack_network_config: Some(RackNetworkConfig { + rack_subnet: "fd00:1122:3344:01::/56".parse().unwrap(), + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports: Vec::new(), + bgp: Vec::new(), + }), }, ) .await diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 34c218b3e2..701a6e8ba9 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -58,6 +58,7 @@ pub const RACK_UUID: &str = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"; pub const SWITCH_UUID: &str = "dae4e1f1-410e-4314-bff1-fec0504be07e"; pub const OXIMETER_UUID: &str = "39e6175b-4df2-4730-b11d-cbc1e60a2e78"; pub const PRODUCER_UUID: &str = "a6458b7d-87c3-4483-be96-854d814c20de"; +pub const RACK_SUBNET: &str = "fd00:1122:3344:01::/56"; /// The reported amount of hardware threads for an emulated sled agent. pub const TEST_HARDWARE_THREADS: u32 = 16; @@ -86,6 +87,7 @@ pub struct ControlPlaneTestContext { pub oximeter: Oximeter, pub producer: ProducerServer, pub dendrite: HashMap, + pub mgd: HashMap, pub external_dns_zone_name: String, pub external_dns: dns_server::TransientServer, pub internal_dns: dns_server::TransientServer, @@ -108,6 +110,9 @@ impl ControlPlaneTestContext { for (_, mut dendrite) in self.dendrite { dendrite.cleanup().await.unwrap(); } + for (_, mut mgd) in self.mgd { + mgd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } } @@ -237,6 +242,7 @@ pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> { pub oximeter: Option, pub producer: Option, pub dendrite: HashMap, + pub mgd: HashMap, // NOTE: Only exists after starting Nexus, until external Nexus is // initialized. @@ -274,6 +280,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { oximeter: None, producer: None, dendrite: HashMap::new(), + mgd: HashMap::new(), nexus_internal: None, nexus_internal_addr: None, external_dns_zone_name: None, @@ -398,6 +405,32 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { ); } + pub async fn start_mgd(&mut self, switch_location: SwitchLocation) { + let log = &self.logctx.log; + debug!(log, "Starting mgd for {switch_location}"); + + // Set up an instance of mgd + let mgd = dev::maghemite::MgdInstance::start(0).await.unwrap(); + let port = mgd.port; + self.mgd.insert(switch_location, mgd); + let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); + + debug!(log, "mgd port is {port}"); + + let config = omicron_common::nexus_config::MgdConfig { + address: std::net::SocketAddr::V6(address), + }; + self.config.pkg.mgd.insert(switch_location, config); + + let sled_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap(); + self.rack_init_builder.add_service( + address, + ServiceKind::Mgd, + internal_dns::ServiceName::Mgd, + sled_id, + ); + } + pub async fn start_oximeter(&mut self) { let log = &self.logctx.log; debug!(log, "Starting Oximeter"); @@ -528,8 +561,11 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { &format!("http://{}", internal_dns_address), log.clone(), ); + let dns_config = self.rack_init_builder.internal_dns_config.clone().build(); + + slog::info!(log, "DNS population: {:#?}", dns_config); dns_config_client.dns_config_put(&dns_config).await.expect( "Failed to send initial DNS records to internal DNS server", ); @@ -669,6 +705,25 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { ); } + pub async fn scrimlet_dns_setup(&mut self) { + let sled_agent = self + .sled_agent + .as_ref() + .expect("Cannot set up scrimlet DNS without sled agent"); + + let sa = match sled_agent.http_server.local_addr() { + SocketAddr::V6(sa) => sa, + SocketAddr::V4(_) => panic!("expected SocketAddrV6 for sled agent"), + }; + + for loc in [SwitchLocation::Switch0, SwitchLocation::Switch1] { + self.rack_init_builder + .internal_dns_config + .host_scrimlet(loc, sa) + .expect("add switch0 scrimlet dns entry"); + } + } + // Set up an external DNS server. pub async fn start_external_dns(&mut self) { let log = self.logctx.log.new(o!("component" => "external_dns_server")); @@ -742,6 +797,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { producer: self.producer.unwrap(), logctx: self.logctx, dendrite: self.dendrite, + mgd: self.mgd, external_dns_zone_name: self.external_dns_zone_name.unwrap(), external_dns: self.external_dns.unwrap(), internal_dns: self.internal_dns.unwrap(), @@ -772,6 +828,9 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { for (_, mut dendrite) in self.dendrite { dendrite.cleanup().await.unwrap(); } + for (_, mut mgd) in self.mgd { + mgd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } } @@ -862,11 +921,14 @@ async fn setup_with_config_impl( builder.start_clickhouse().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.start_internal_dns().await; builder.start_external_dns().await; builder.start_nexus_internal().await; builder.start_sled(sim_mode).await; builder.start_crucible_pantry().await; + builder.scrimlet_dns_setup().await; // Give Nexus necessary information to find the Crucible Pantry let dns_config = builder.populate_internal_dns().await; diff --git a/nexus/tests/integration_tests/address_lots.rs b/nexus/tests/integration_tests/address_lots.rs index b4659daa62..40c8865929 100644 --- a/nexus/tests/integration_tests/address_lots.rs +++ b/nexus/tests/integration_tests/address_lots.rs @@ -27,8 +27,8 @@ type ControlPlaneTestContext = async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { let client = &ctx.external_client; - // Verify there are no lots - let lots = NexusRequest::iter_collection_authn::( + // Verify there is only one system lot + let lots = NexusRequest::iter_collection_authn::( client, "/v1/system/networking/address-lot", "", @@ -37,7 +37,7 @@ async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { .await .expect("Failed to list address lots") .all_items; - assert_eq!(lots.len(), 0, "Expected no lots"); + assert_eq!(lots.len(), 1, "Expected one lot"); // Create a lot let params = AddressLotCreate { @@ -111,8 +111,8 @@ async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { .expect("Failed to list address lots") .all_items; - assert_eq!(lots.len(), 1, "Expected 1 lot"); - assert_eq!(lots[0], address_lot); + assert_eq!(lots.len(), 2, "Expected 2 lots"); + assert_eq!(lots[1], address_lot); // Verify there are lot blocks let blist = NexusRequest::iter_collection_authn::( diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index e9ae11c21f..8fba22fb2f 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -420,6 +420,40 @@ lazy_static! { }; } +lazy_static! { + pub static ref DEMO_BGP_CONFIG_CREATE_URL: String = + format!("/v1/system/networking/bgp?name_or_id=as47"); + pub static ref DEMO_BGP_CONFIG: params::BgpConfigCreate = + params::BgpConfigCreate { + identity: IdentityMetadataCreateParams { + name: "as47".parse().unwrap(), + description: "BGP config for AS47".into(), + }, + bgp_announce_set_id: NameOrId::Name("instances".parse().unwrap()), + asn: 47, + vrf: None, + }; + pub static ref DEMO_BGP_ANNOUNCE_SET_URL: String = + format!("/v1/system/networking/bgp-announce?name_or_id=a-bag-of-addrs"); + pub static ref DEMO_BGP_ANNOUNCE: params::BgpAnnounceSetCreate = + params::BgpAnnounceSetCreate { + identity: IdentityMetadataCreateParams { + name: "a-bag-of-addrs".parse().unwrap(), + description: "a bag of addrs".into(), + }, + announcement: vec![params::BgpAnnouncementCreate { + address_lot_block: NameOrId::Name( + "some-block".parse().unwrap(), + ), + network: "10.0.0.0/16".parse().unwrap(), + }], + }; + pub static ref DEMO_BGP_STATUS_URL: String = + format!("/v1/system/networking/bgp-status"); + pub static ref DEMO_BGP_ROUTES_IPV4_URL: String = + format!("/v1/system/networking/bgp-routes-ipv4?asn=47"); +} + lazy_static! { // Project Images pub static ref DEMO_IMAGE_NAME: Name = "demo-image".parse().unwrap(); @@ -1876,5 +1910,48 @@ lazy_static! { AllowedMethod::GetNonexistent ], }, + VerifyEndpoint { + url: &DEMO_BGP_CONFIG_CREATE_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_BGP_CONFIG).unwrap(), + ), + AllowedMethod::Get, + AllowedMethod::Delete + ], + }, + + VerifyEndpoint { + url: &DEMO_BGP_ANNOUNCE_SET_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_BGP_ANNOUNCE).unwrap(), + ), + AllowedMethod::GetNonexistent, + AllowedMethod::Delete + ], + }, + + VerifyEndpoint { + url: &DEMO_BGP_STATUS_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + ], + }, + + VerifyEndpoint { + url: &DEMO_BGP_ROUTES_IPV4_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + ], + } ]; } diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index 2d4c76dc99..43a4ac8f2e 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -29,6 +29,8 @@ async fn test_nexus_boots_before_cockroach() { builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.start_internal_dns().await; builder.start_external_dns().await; @@ -144,6 +146,11 @@ async fn test_nexus_boots_before_dendrite() { builder.start_dendrite(SwitchLocation::Switch1).await; info!(log, "Started Dendrite"); + info!(log, "Starting mgd"); + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; + info!(log, "Started mgd"); + info!(log, "Populating internal DNS records"); builder.populate_internal_dns().await; info!(log, "Populated internal DNS records"); @@ -166,6 +173,8 @@ async fn nexus_schema_test_setup( builder.start_external_dns().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.populate_internal_dns().await; } diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index f7d6c1da6a..e75211b834 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -58,6 +58,8 @@ async fn test_setup<'a>( builder.start_external_dns().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.populate_internal_dns().await; builder } diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index 3d3d6c9f5f..fada45694d 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -10,8 +10,10 @@ use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder}; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params::{ Address, AddressConfig, AddressLotBlockCreate, AddressLotCreate, - LinkConfig, LldpServiceConfig, Route, RouteConfig, SwitchInterfaceConfig, - SwitchInterfaceKind, SwitchPortApplySettings, SwitchPortSettingsCreate, + BgpAnnounceSetCreate, BgpAnnouncementCreate, BgpConfigCreate, + BgpPeerConfig, LinkConfig, LinkFec, LinkSpeed, LldpServiceConfig, Route, + RouteConfig, SwitchInterfaceConfig, SwitchInterfaceKind, + SwitchPortApplySettings, SwitchPortSettingsCreate, }; use nexus_types::external_api::views::Rack; use omicron_common::api::external::{ @@ -33,10 +35,16 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { description: "an address parking lot".into(), }, kind: AddressLotKind::Infra, - blocks: vec![AddressLotBlockCreate { - first_address: "203.0.113.10".parse().unwrap(), - last_address: "203.0.113.20".parse().unwrap(), - }], + blocks: vec![ + AddressLotBlockCreate { + first_address: "203.0.113.10".parse().unwrap(), + last_address: "203.0.113.20".parse().unwrap(), + }, + AddressLotBlockCreate { + first_address: "1.2.3.0".parse().unwrap(), + last_address: "1.2.3.255".parse().unwrap(), + }, + ], }; NexusRequest::objects_post( @@ -49,6 +57,49 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .await .unwrap(); + // Create BGP announce set + let announce_set = BgpAnnounceSetCreate { + identity: IdentityMetadataCreateParams { + name: "instances".parse().unwrap(), + description: "autonomous system 47 announcements".into(), + }, + announcement: vec![BgpAnnouncementCreate { + address_lot_block: NameOrId::Name("parkinglot".parse().unwrap()), + network: "1.2.3.0/24".parse().unwrap(), + }], + }; + + NexusRequest::objects_post( + client, + "/v1/system/networking/bgp-announce", + &announce_set, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + + // Create BGP config + let bgp_config = BgpConfigCreate { + identity: IdentityMetadataCreateParams { + name: "as47".parse().unwrap(), + description: "autonomous system 47".into(), + }, + bgp_announce_set_id: NameOrId::Name("instances".parse().unwrap()), + asn: 47, + vrf: None, + }; + + NexusRequest::objects_post( + client, + "/v1/system/networking/bgp", + &bgp_config, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + // Create port settings let mut settings = SwitchPortSettingsCreate::new(IdentityMetadataCreateParams { @@ -61,6 +112,8 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { LinkConfig { mtu: 4700, lldp: LldpServiceConfig { enabled: false, lldp_config: None }, + fec: LinkFec::None, + speed: LinkSpeed::Speed100G, }, ); // interfaces @@ -191,6 +244,33 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .parsed_body() .unwrap(); + // Update port settings. Should not see conflict. + settings.bgp_peers.insert( + "phy0".into(), + BgpPeerConfig { + bgp_config: NameOrId::Name("as47".parse().unwrap()), //TODO + bgp_announce_set: NameOrId::Name("instances".parse().unwrap()), //TODO + interface_name: "phy0".to_string(), + addr: "1.2.3.4".parse().unwrap(), + hold_time: 6, + idle_hold_time: 6, + delay_open: 0, + connect_retry: 3, + keepalive: 2, + }, + ); + let _created: SwitchPortSettingsView = NexusRequest::objects_post( + client, + "/v1/system/networking/switch-port-settings", + &settings, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + // There should be one switch port to begin with, see // Server::start_and_populate in nexus/src/lib.rs diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 1d7f5556c2..e55eaa4df6 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -145,6 +145,14 @@ networking_address_lot_block_list GET /v1/system/networking/address- networking_address_lot_create POST /v1/system/networking/address-lot networking_address_lot_delete DELETE /v1/system/networking/address-lot/{address_lot} networking_address_lot_list GET /v1/system/networking/address-lot +networking_bgp_announce_set_create POST /v1/system/networking/bgp-announce +networking_bgp_announce_set_delete DELETE /v1/system/networking/bgp-announce +networking_bgp_announce_set_list GET /v1/system/networking/bgp-announce +networking_bgp_config_create POST /v1/system/networking/bgp +networking_bgp_config_delete DELETE /v1/system/networking/bgp +networking_bgp_config_list GET /v1/system/networking/bgp +networking_bgp_imported_routes_ipv4 GET /v1/system/networking/bgp-routes-ipv4 +networking_bgp_status GET /v1/system/networking/bgp-status networking_loopback_address_create POST /v1/system/networking/loopback-address networking_loopback_address_delete DELETE /v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask} networking_loopback_address_list GET /v1/system/networking/loopback-address diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index b4e0e705d8..a0169ae777 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -1325,6 +1325,42 @@ pub enum SwitchPortGeometry { Sfp28x4, } +/// The forward error correction mode of a link. +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LinkFec { + /// Firecode foward error correction. + Firecode, + /// No forward error correction. + None, + /// Reed-Solomon forward error correction. + Rs, +} + +/// The speed of a link. +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LinkSpeed { + /// Zero gigabits per second. + Speed0G, + /// 1 gigabit per second. + Speed1G, + /// 10 gigabits per second. + Speed10G, + /// 25 gigabits per second. + Speed25G, + /// 40 gigabits per second. + Speed40G, + /// 50 gigabits per second. + Speed50G, + /// 100 gigabits per second. + Speed100G, + /// 200 gigabits per second. + Speed200G, + /// 400 gigabits per second. + Speed400G, +} + /// Switch link configuration. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct LinkConfig { @@ -1333,6 +1369,12 @@ pub struct LinkConfig { /// The link-layer discovery protocol (LLDP) configuration for the link. pub lldp: LldpServiceConfig, + + /// The forward error correction mode of the link. + pub fec: LinkFec, + + /// The speed of the link. + pub speed: LinkSpeed, } /// The LLDP configuration associated with a port. LLDP may be either enabled or @@ -1406,6 +1448,20 @@ pub struct Route { pub vid: Option, } +/// Select a BGP config by a name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpConfigSelector { + /// A name or id to use when selecting BGP config. + pub name_or_id: NameOrId, +} + +/// List BGP configs with an optional name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpConfigListSelector { + /// A name or id to use when selecting BGP config. + pub name_or_id: Option, +} + /// A BGP peer configuration for an interface. Includes the set of announcements /// that will be advertised to the peer identified by `addr`. The `bgp_config` /// parameter is a reference to global BGP parameters. The `interface_name` @@ -1427,21 +1483,59 @@ pub struct BgpPeerConfig { /// The address of the host to peer with. pub addr: IpAddr, + + /// How long to hold peer connections between keppalives (seconds). + pub hold_time: u32, + + /// How long to hold a peer in idle before attempting a new session + /// (seconds). + pub idle_hold_time: u32, + + /// How long to delay sending an open request after establishing a TCP + /// session (seconds). + pub delay_open: u32, + + /// How long to to wait between TCP connection retries (seconds). + pub connect_retry: u32, + + /// How often to send keepalive requests (seconds). + pub keepalive: u32, } /// Parameters for creating a named set of BGP announcements. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct CreateBgpAnnounceSet { +pub struct BgpAnnounceSetCreate { #[serde(flatten)] pub identity: IdentityMetadataCreateParams, /// The announcements in this set. - pub announcement: Vec, + pub announcement: Vec, +} + +/// Select a BGP announce set by a name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpAnnounceSetSelector { + /// A name or id to use when selecting BGP port settings + pub name_or_id: NameOrId, +} + +/// List BGP announce set with an optional name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpAnnounceListSelector { + /// A name or id to use when selecting BGP config. + pub name_or_id: Option, +} + +/// Selector used for querying imported BGP routes. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpRouteSelector { + /// The ASN to filter on. Required. + pub asn: u32, } /// A BGP announcement tied to a particular address lot block. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct BgpAnnouncement { +pub struct BgpAnnouncementCreate { /// Address lot this announcement is drawn from. pub address_lot_block: NameOrId, @@ -1452,18 +1546,27 @@ pub struct BgpAnnouncement { /// Parameters for creating a BGP configuration. This includes and autonomous /// system number (ASN) and a virtual routing and forwarding (VRF) identifier. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct CreateBgpConfig { +pub struct BgpConfigCreate { #[serde(flatten)] pub identity: IdentityMetadataCreateParams, /// The autonomous system number of this BGP configuration. pub asn: u32, + pub bgp_announce_set_id: NameOrId, + /// Optional virtual routing and forwarding identifier for this BGP /// configuration. pub vrf: Option, } +/// Select a BGP status information by BGP config id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpStatusSelector { + /// A name or id of the BGP configuration to get status for + pub name_or_id: NameOrId, +} + /// A set of addresses associated with a port configuration. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct AddressConfig { diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index e2a5e3d094..c0991ebb17 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -182,6 +182,7 @@ pub enum ServiceKind { Tfport, BoundaryNtp { snat: SourceNatConfig, nic: ServiceNic }, InternalNtp, + Mgd, } impl fmt::Display for ServiceKind { @@ -200,6 +201,7 @@ impl fmt::Display for ServiceKind { Tfport => "tfport", CruciblePantry => "crucible_pantry", BoundaryNtp { .. } | InternalNtp => "ntp", + Mgd => "mgd", }; write!(f, "{}", s) } diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 682512cc24..6dcf756737 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -241,6 +241,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BootstrapAddressDiscovery": { "oneOf": [ { @@ -333,6 +380,26 @@ "request_id" ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -375,6 +442,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -406,6 +477,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -492,7 +626,7 @@ "description": "Initial rack network configuration", "allOf": [ { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] }, @@ -529,10 +663,17 @@ "recovery_silo" ] }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -543,18 +684,23 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ] }, "RackOperationStatus": { @@ -747,6 +893,28 @@ "user_password_hash" ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -770,67 +938,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "UserId": { "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", "type": "string" diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 67db222155..411c52ddff 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -767,6 +767,53 @@ "serial_number" ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BinRangedouble": { "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", "oneOf": [ @@ -3653,6 +3700,26 @@ } ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -3695,6 +3762,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -4038,6 +4109,69 @@ "PhysicalDiskPutResponse": { "type": "object" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -4268,7 +4402,7 @@ "description": "Initial rack network configuration", "allOf": [ { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] }, @@ -4299,10 +4433,17 @@ "services" ] }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -4313,18 +4454,23 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ] }, "RecoverySiloConfig": { @@ -4346,6 +4492,28 @@ "user_password_hash" ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "Saga": { "description": "Sagas\n\nThese are currently only intended for observability by developers. We will eventually want to flesh this out into something more observable for end users.", "type": "object", @@ -4822,6 +4990,20 @@ "required": [ "type" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "mgd" + ] + } + }, + "required": [ + "type" + ] } ] }, @@ -5090,67 +5272,6 @@ "SwitchPutResponse": { "type": "object" }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "UserId": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", diff --git a/openapi/nexus.json b/openapi/nexus.json index 9dda94f283..456f2aebd6 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -5165,6 +5165,318 @@ } } }, + "/v1/system/networking/bgp": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get BGP configurations.", + "operationId": "networking_bgp_config_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP config.", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpConfigResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + }, + "post": { + "tags": [ + "system/networking" + ], + "summary": "Create a new BGP configuration.", + "operationId": "networking_bgp_config_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpConfigCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "system/networking" + ], + "summary": "Delete a BGP configuration.", + "operationId": "networking_bgp_config_delete", + "parameters": [ + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP config.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/networking/bgp-announce": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get originated routes for a given BGP configuration.", + "operationId": "networking_bgp_announce_set_list", + "parameters": [ + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP port settings", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_BgpAnnouncement", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpAnnouncement" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "system/networking" + ], + "summary": "Create a new BGP announce set.", + "operationId": "networking_bgp_announce_set_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpAnnounceSetCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpAnnounceSet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "system/networking" + ], + "summary": "Delete a BGP announce set.", + "operationId": "networking_bgp_announce_set_delete", + "parameters": [ + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP port settings", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/networking/bgp-routes-ipv4": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get imported IPv4 BGP routes.", + "operationId": "networking_bgp_imported_routes_ipv4", + "parameters": [ + { + "in": "query", + "name": "asn", + "description": "The ASN to filter on. Required.", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_BgpImportedRouteIpv4", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpImportedRouteIpv4" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/networking/bgp-status": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get BGP peer status", + "operationId": "networking_bgp_status", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_BgpPeerStatus", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerStatus" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/system/networking/loopback-address": { "get": { "tags": [ @@ -7741,40 +8053,289 @@ "$ref": "#/components/schemas/AddressLotBlock" } }, - "lot": { - "description": "The address lot that was created.", + "lot": { + "description": "The address lot that was created.", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLot" + } + ] + } + }, + "required": [ + "blocks", + "lot" + ] + }, + "AddressLotKind": { + "description": "The kind associated with an address lot.", + "oneOf": [ + { + "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", + "type": "string", + "enum": [ + "infra" + ] + }, + { + "description": "Pool address lots are used by IP pools.", + "type": "string", + "enum": [ + "pool" + ] + } + ] + }, + "AddressLotResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLot" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "Baseboard": { + "description": "Properties that uniquely identify an Oxide hardware component", + "type": "object", + "properties": { + "part": { + "type": "string" + }, + "revision": { + "type": "integer", + "format": "int64" + }, + "serial": { + "type": "string" + } + }, + "required": [ + "part", + "revision", + "serial" + ] + }, + "BgpAnnounceSet": { + "description": "Represents a BGP announce set by id. The id can be used with other API calls to view and manage the announce set.", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "id", + "name", + "time_created", + "time_modified" + ] + }, + "BgpAnnounceSetCreate": { + "description": "Parameters for creating a named set of BGP announcements.", + "type": "object", + "properties": { + "announcement": { + "description": "The announcements in this set.", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpAnnouncementCreate" + } + }, + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "announcement", + "description", + "name" + ] + }, + "BgpAnnouncement": { + "description": "A BGP announcement tied to an address lot block.", + "type": "object", + "properties": { + "address_lot_block_id": { + "description": "The address block the IP network being announced is drawn from.", + "type": "string", + "format": "uuid" + }, + "announce_set_id": { + "description": "The id of the set this announcement is a part of.", + "type": "string", + "format": "uuid" + }, + "network": { + "description": "The IP network being announced.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + } + }, + "required": [ + "address_lot_block_id", + "announce_set_id", + "network" + ] + }, + "BgpAnnouncementCreate": { + "description": "A BGP announcement tied to a particular address lot block.", + "type": "object", + "properties": { + "address_lot_block": { + "description": "Address lot this announcement is drawn from.", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + }, + "network": { + "description": "The network being announced.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + } + }, + "required": [ + "address_lot_block", + "network" + ] + }, + "BgpConfig": { + "description": "A base BGP configuration.", + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number of this BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + }, + "vrf": { + "nullable": true, + "description": "Optional virtual routing and forwarding identifier for this BGP configuration.", + "type": "string" + } + }, + "required": [ + "asn", + "description", + "id", + "name", + "time_created", + "time_modified" + ] + }, + "BgpConfigCreate": { + "description": "Parameters for creating a BGP configuration. This includes and autonomous system number (ASN) and a virtual routing and forwarding (VRF) identifier.", + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number of this BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "bgp_announce_set_id": { + "$ref": "#/components/schemas/NameOrId" + }, + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "vrf": { + "nullable": true, + "description": "Optional virtual routing and forwarding identifier for this BGP configuration.", "allOf": [ { - "$ref": "#/components/schemas/AddressLot" + "$ref": "#/components/schemas/Name" } ] } }, "required": [ - "blocks", - "lot" - ] - }, - "AddressLotKind": { - "description": "The kind associated with an address lot.", - "oneOf": [ - { - "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", - "type": "string", - "enum": [ - "infra" - ] - }, - { - "description": "Pool address lots are used by IP pools.", - "type": "string", - "enum": [ - "pool" - ] - } + "asn", + "bgp_announce_set_id", + "description", + "name" ] }, - "AddressLotResultsPage": { + "BgpConfigResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -7782,7 +8343,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/AddressLot" + "$ref": "#/components/schemas/BgpConfig" } }, "next_page": { @@ -7795,25 +8356,43 @@ "items" ] }, - "Baseboard": { - "description": "Properties that uniquely identify an Oxide hardware component", + "BgpImportedRouteIpv4": { + "description": "A route imported from a BGP peer.", "type": "object", "properties": { - "part": { - "type": "string" - }, - "revision": { + "id": { + "description": "BGP identifier of the originating router.", "type": "integer", - "format": "int64" + "format": "uint32", + "minimum": 0 }, - "serial": { - "type": "string" + "nexthop": { + "description": "The nexthop the prefix is reachable through.", + "type": "string", + "format": "ipv4" + }, + "prefix": { + "description": "The destination network prefix.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Net" + } + ] + }, + "switch": { + "description": "Switch the route is imported into.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] } }, "required": [ - "part", - "revision", - "serial" + "id", + "nexthop", + "prefix", + "switch" ] }, "BgpPeerConfig": { @@ -7841,16 +8420,158 @@ } ] }, + "connect_retry": { + "description": "How long to to wait between TCP connection retries (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "delay_open": { + "description": "How long to delay sending an open request after establishing a TCP session (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "hold_time": { + "description": "How long to hold peer connections between keppalives (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "idle_hold_time": { + "description": "How long to hold a peer in idle before attempting a new session (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "interface_name": { "description": "The name of interface to peer on. This is relative to the port configuration this BGP peer configuration is a part of. For example this value could be phy0 to refer to a primary physical interface. Or it could be vlan47 to refer to a VLAN interface.", "type": "string" + }, + "keepalive": { + "description": "How often to send keepalive requests (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 } }, "required": [ "addr", "bgp_announce_set", "bgp_config", - "interface_name" + "connect_retry", + "delay_open", + "hold_time", + "idle_hold_time", + "interface_name", + "keepalive" + ] + }, + "BgpPeerState": { + "description": "The current state of a BGP peer.", + "oneOf": [ + { + "description": "Initial state. Refuse all incomming BGP connections. No resources allocated to peer.", + "type": "string", + "enum": [ + "idle" + ] + }, + { + "description": "Waiting for the TCP connection to be completed.", + "type": "string", + "enum": [ + "connect" + ] + }, + { + "description": "Trying to acquire peer by listening for and accepting a TCP connection.", + "type": "string", + "enum": [ + "active" + ] + }, + { + "description": "Waiting for open message from peer.", + "type": "string", + "enum": [ + "open_sent" + ] + }, + { + "description": "Waiting for keepaliave or notification from peer.", + "type": "string", + "enum": [ + "open_confirm" + ] + }, + { + "description": "Synchronizing with peer.", + "type": "string", + "enum": [ + "session_setup" + ] + }, + { + "description": "Session established. Able to exchange update, notification and keepliave messages with peers.", + "type": "string", + "enum": [ + "established" + ] + } + ] + }, + "BgpPeerStatus": { + "description": "The current status of a BGP peer.", + "type": "object", + "properties": { + "addr": { + "description": "IP address of the peer.", + "type": "string", + "format": "ip" + }, + "local_asn": { + "description": "Local autonomous system number.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "remote_asn": { + "description": "Remote autonomous system number.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "state": { + "description": "State of the peer.", + "allOf": [ + { + "$ref": "#/components/schemas/BgpPeerState" + } + ] + }, + "state_duration_millis": { + "description": "Time of last state change.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "switch": { + "description": "Switch with the peer session.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + } + }, + "required": [ + "addr", + "local_asn", + "remote_asn", + "state", + "state_duration_millis", + "switch" ] }, "BinRangedouble": { @@ -11747,6 +12468,14 @@ "description": "Switch link configuration.", "type": "object", "properties": { + "fec": { + "description": "The forward error correction mode of the link.", + "allOf": [ + { + "$ref": "#/components/schemas/LinkFec" + } + ] + }, "lldp": { "description": "The link-layer discovery protocol (LLDP) configuration for the link.", "allOf": [ @@ -11760,11 +12489,115 @@ "type": "integer", "format": "uint16", "minimum": 0 + }, + "speed": { + "description": "The speed of the link.", + "allOf": [ + { + "$ref": "#/components/schemas/LinkSpeed" + } + ] } }, "required": [ + "fec", "lldp", - "mtu" + "mtu", + "speed" + ] + }, + "LinkFec": { + "description": "The forward error correction mode of a link.", + "oneOf": [ + { + "description": "Firecode foward error correction.", + "type": "string", + "enum": [ + "firecode" + ] + }, + { + "description": "No forward error correction.", + "type": "string", + "enum": [ + "none" + ] + }, + { + "description": "Reed-Solomon forward error correction.", + "type": "string", + "enum": [ + "rs" + ] + } + ] + }, + "LinkSpeed": { + "description": "The speed of a link.", + "oneOf": [ + { + "description": "Zero gigabits per second.", + "type": "string", + "enum": [ + "speed0_g" + ] + }, + { + "description": "1 gigabit per second.", + "type": "string", + "enum": [ + "speed1_g" + ] + }, + { + "description": "10 gigabits per second.", + "type": "string", + "enum": [ + "speed10_g" + ] + }, + { + "description": "25 gigabits per second.", + "type": "string", + "enum": [ + "speed25_g" + ] + }, + { + "description": "40 gigabits per second.", + "type": "string", + "enum": [ + "speed40_g" + ] + }, + { + "description": "50 gigabits per second.", + "type": "string", + "enum": [ + "speed50_g" + ] + }, + { + "description": "100 gigabits per second.", + "type": "string", + "enum": [ + "speed100_g" + ] + }, + { + "description": "200 gigabits per second.", + "type": "string", + "enum": [ + "speed200_g" + ] + }, + { + "description": "400 gigabits per second.", + "type": "string", + "enum": [ + "speed400_g" + ] + } ] }, "LldpServiceConfig": { @@ -13539,6 +14372,25 @@ } ] }, + "SwitchLocation": { + "description": "Identifies switch physical location", + "oneOf": [ + { + "description": "Switch in upper slot", + "type": "string", + "enum": [ + "switch0" + ] + }, + { + "description": "Switch in lower slot", + "type": "string", + "enum": [ + "switch1" + ] + } + ] + }, "SwitchPort": { "description": "A switch port represents a physical external port on a rack switch.", "type": "object", @@ -13635,11 +14487,6 @@ "type": "string", "format": "ip" }, - "bgp_announce_set_id": { - "description": "The id for the set of prefixes announced in this peer configuration.", - "type": "string", - "format": "uuid" - }, "bgp_config_id": { "description": "The id of the global BGP configuration referenced by this peer configuration.", "type": "string", @@ -13657,7 +14504,6 @@ }, "required": [ "addr", - "bgp_announce_set_id", "bgp_config_id", "interface_name", "port_settings_id" diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 7831193fc2..486662853c 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -289,6 +289,55 @@ } } }, + "/network-bootstore-config": { + "get": { + "summary": "This API endpoint is only reading the local sled agent's view of the", + "description": "bootstore. The boostore is a distributed data store that is eventually consistent. Reads from individual nodes may not represent the latest state.", + "operationId": "read_network_bootstore_config_cache", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "write_network_bootstore_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/services": { "put": { "operationId": "services_put", @@ -338,6 +387,32 @@ } } }, + "/switch-ports": { + "post": { + "operationId": "uplink_ensure", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SwitchPorts" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/timesync": { "get": { "operationId": "timesync_get", @@ -863,6 +938,53 @@ } }, "schemas": { + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BundleUtilization": { "description": "The portion of a debug dataset used for zone bundles.", "type": "object", @@ -1601,6 +1723,54 @@ "secs" ] }, + "EarlyNetworkConfig": { + "description": "Network configuration required to bring up the control plane\n\nThe fields in this structure are those from [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This is just for the initial rack configuration and cold boot purposes. Updates come from Nexus.", + "type": "object", + "properties": { + "body": { + "$ref": "#/components/schemas/EarlyNetworkConfigBody" + }, + "generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "body", + "generation", + "schema_version" + ] + }, + "EarlyNetworkConfigBody": { + "description": "This is the actual configuration of EarlyNetworking.\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "type": "object", + "properties": { + "ntp_servers": { + "description": "The external NTP server addresses.", + "type": "array", + "items": { + "type": "string" + } + }, + "rack_network_config": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RackNetworkConfigV1" + } + ] + } + }, + "required": [ + "ntp_servers" + ] + }, "Error": { "description": "Error information from a response.", "type": "object", @@ -1667,6 +1837,26 @@ } ] }, + "HostPortConfig": { + "type": "object", + "properties": { + "addrs": { + "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "port": { + "description": "Switchport to use for external connectivity", + "type": "string" + } + }, + "required": [ + "addrs", + "port" + ] + }, "InstanceCpuCount": { "description": "The number of CPUs in an Instance", "type": "integer", @@ -2133,6 +2323,26 @@ } ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "Ipv4Net": { "example": "192.168.1.0/24", "title": "An IPv4 subnet", @@ -2140,6 +2350,10 @@ "type": "string", "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$" }, + "Ipv4Network": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\/(3[0-2]|[0-2]?[0-9])$" + }, "Ipv6Net": { "example": "fd12:3456::/64", "title": "An IPv6 subnet", @@ -2147,6 +2361,10 @@ "type": "string", "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "KnownArtifactKind": { "description": "Kinds of update artifacts, as used by Nexus to determine what updates are available and by sled-agent to determine how to apply an update when asked.", "type": "string", @@ -2281,6 +2499,93 @@ } ] }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, + "PortFec": { + "description": "Switchport FEC options", + "type": "string", + "enum": [ + "firecode", + "none", + "rs" + ] + }, + "PortSpeed": { + "description": "Switchport Speed options", + "type": "string", + "enum": [ + "speed0_g", + "speed1_g", + "speed10_g", + "speed25_g", + "speed40_g", + "speed50_g", + "speed100_g", + "speed200_g", + "speed400_g" + ] + }, "PriorityDimension": { "description": "A dimension along with bundles can be sorted, to determine priority.", "oneOf": [ @@ -2309,6 +2614,68 @@ "minItems": 2, "maxItems": 2 }, + "RackNetworkConfigV1": { + "description": "Initial network configuration", + "type": "object", + "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, + "infra_ip_first": { + "description": "First ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "infra_ip_last": { + "description": "Last ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "ports": { + "description": "Uplinks for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/PortConfigV1" + } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" + } + }, + "required": [ + "bgp", + "infra_ip_first", + "infra_ip_last", + "ports", + "rack_subnet" + ] + }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -2823,6 +3190,40 @@ "format": "uint8", "minimum": 0 }, + "SwitchLocation": { + "description": "Identifies switch physical location", + "oneOf": [ + { + "description": "Switch in upper slot", + "type": "string", + "enum": [ + "switch0" + ] + }, + { + "description": "Switch in lower slot", + "type": "string", + "enum": [ + "switch1" + ] + } + ] + }, + "SwitchPorts": { + "description": "A set of switch uplinks.", + "type": "object", + "properties": { + "uplinks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HostPortConfig" + } + } + }, + "required": [ + "uplinks" + ] + }, "TimeSync": { "type": "object", "properties": { diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 8b4da8970f..75db82e8e1 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -838,6 +838,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BootstrapSledDescription": { "type": "object", "properties": { @@ -1025,7 +1072,7 @@ "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] } @@ -1339,6 +1386,26 @@ "installable" ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -1381,6 +1448,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -1404,6 +1475,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -1973,7 +2107,7 @@ } }, "rack_network_config": { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } }, "required": [ @@ -1990,10 +2124,17 @@ "type": "string", "format": "uuid" }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -2004,18 +2145,23 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ] }, "RackOperationStatus": { @@ -2332,6 +2478,28 @@ } ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -4457,67 +4625,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "IgnitionCommand": { "description": "Ignition command.", "type": "string", diff --git a/package-manifest.toml b/package-manifest.toml index a88f8170d0..3404f5f44f 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -412,7 +412,7 @@ source.commit = "334df299a56cd0d33e1227ed4ce4d2fe7478d541" source.sha256 = "531e0654de94b6e805836c35aa88b8a1ac691184000a03976e2b7825061e904e" output.type = "zone" -[package.maghemite] +[package.mg-ddm-gz] service_name = "mg-ddm" # Note: unlike every other package, `maghemite` is not restricted to either the # "standard" or "trampoline" image; it is included in both. @@ -422,10 +422,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "12703675393459e74139f8140e0b3c4c4f129d5d" +source.commit = "2f25a2005521f643317879b46692141b4127608a" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt -source.sha256 = "e57fe791ee898d59890c5779fbd4dce598250fb6ed53832024212bcdeec0cc5b" +source.sha256 = "e808388cd080a3325fb5429314a5674809bcde24ad0456b58b57c87cbaa1300d" output.type = "tarball" [package.mg-ddm] @@ -438,10 +438,25 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "12703675393459e74139f8140e0b3c4c4f129d5d" +source.commit = "2f25a2005521f643317879b46692141b4127608a" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "3aa0d32b1d2b6be7091b9c665657296e924a86a00ca38756e9f45a1e629fd92b" +source.sha256 = "27e4845fd11b9768559eb9835309387e83c95628a6a292977e734e8bc7f9fa0f" +output.type = "zone" +output.intermediate_only = true + +[package.mgd] +service_name = "mgd" +source.type = "prebuilt" +source.repo = "maghemite" +# Updating the commit hash here currently requires also updating +# `tools/maghemite_openapi_version`. Failing to do so will cause a failure when +# building `ddm-admin-client` (which will instruct you to update +# `tools/maghemite_openapi_version`). +source.commit = "2f25a2005521f643317879b46692141b4127608a" +# The SHA256 digest is automatically posted to: +# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt +source.sha256 = "16878501f5440590674acd82bee6ce5dcf3d1326531c25064dd9c060ab6440a4" output.type = "zone" output.intermediate_only = true @@ -458,8 +473,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "7712104585266a2898da38c1345210ad26f9e71d" -source.sha256 = "486b0b016c0df06947810b90f3a3dd40423f0ee6f255ed079dc8e5618c9a7281" +source.commit = "c0cbc39b55fac54b95468304c497e00f3d3cf686" +source.sha256 = "3706e0e8230b7f76407ec0acea9020b9efc7d6c78b74c304102fd8e62cac6760" output.type = "zone" output.intermediate_only = true @@ -483,8 +498,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "7712104585266a2898da38c1345210ad26f9e71d" -source.sha256 = "76ff76d3526323c3fcbe2351cf9fbda4840e0dc11cd0eb6b71a3e0bd36c5e5e8" +source.commit = "c0cbc39b55fac54b95468304c497e00f3d3cf686" +source.sha256 = "f0847927f7d7197d9a5c4267a0bd0af609d18fd8d6d9b80755c370872c5297fa" output.type = "zone" output.intermediate_only = true @@ -501,8 +516,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "7712104585266a2898da38c1345210ad26f9e71d" -source.sha256 = "b8e5c176070f9bc9ea0028de1999c77d66ea3438913664163975964effe4481b" +source.commit = "c0cbc39b55fac54b95468304c497e00f3d3cf686" +source.sha256 = "33b5897db1fe7b57d282531724ecd7bf74f5156f9aa23f10c6f0d9b54c38a987" output.type = "zone" output.intermediate_only = true @@ -534,6 +549,7 @@ source.packages = [ "wicketd.tar.gz", "wicket.tar.gz", "mg-ddm.tar.gz", + "mgd.tar.gz", "switch_zone_setup.tar.gz", "xcvradm.tar.gz" ] @@ -555,6 +571,7 @@ source.packages = [ "wicketd.tar.gz", "wicket.tar.gz", "mg-ddm.tar.gz", + "mgd.tar.gz", "switch_zone_setup.tar.gz", "sp-sim-stub.tar.gz" ] @@ -576,6 +593,7 @@ source.packages = [ "wicketd.tar.gz", "wicket.tar.gz", "mg-ddm.tar.gz", + "mgd.tar.gz", "switch_zone_setup.tar.gz", "sp-sim-softnpu.tar.gz" ] diff --git a/schema/crdb/8.0.0/up01.sql b/schema/crdb/8.0.0/up01.sql new file mode 100644 index 0000000000..c617a0b634 --- /dev/null +++ b/schema/crdb/8.0.0/up01.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.service_kind ADD VALUE IF NOT EXISTS 'mgd'; diff --git a/schema/crdb/8.0.0/up02.sql b/schema/crdb/8.0.0/up02.sql new file mode 100644 index 0000000000..119e7b9a86 --- /dev/null +++ b/schema/crdb/8.0.0/up02.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.bgp_config ADD COLUMN IF NOT EXISTS bgp_announce_set_id UUID NOT NULL; diff --git a/schema/crdb/8.0.0/up03.sql b/schema/crdb/8.0.0/up03.sql new file mode 100644 index 0000000000..3705d4091e --- /dev/null +++ b/schema/crdb/8.0.0/up03.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config DROP COLUMN IF EXISTS bgp_announce_set_id; diff --git a/schema/crdb/8.0.0/up04.sql b/schema/crdb/8.0.0/up04.sql new file mode 100644 index 0000000000..c5a91796dd --- /dev/null +++ b/schema/crdb/8.0.0/up04.sql @@ -0,0 +1,5 @@ +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM ( + 'Firecode', + 'None', + 'Rs' +); diff --git a/schema/crdb/8.0.0/up05.sql b/schema/crdb/8.0.0/up05.sql new file mode 100644 index 0000000000..4d94bafb9f --- /dev/null +++ b/schema/crdb/8.0.0/up05.sql @@ -0,0 +1,11 @@ +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( + '0G', + '1G', + '10G', + '25G', + '40G', + '50G', + '100G', + '200G', + '400G' +); diff --git a/schema/crdb/8.0.0/up06.sql b/schema/crdb/8.0.0/up06.sql new file mode 100644 index 0000000000..e27800969c --- /dev/null +++ b/schema/crdb/8.0.0/up06.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; diff --git a/schema/crdb/8.0.0/up07.sql b/schema/crdb/8.0.0/up07.sql new file mode 100644 index 0000000000..c84ae8e5d2 --- /dev/null +++ b/schema/crdb/8.0.0/up07.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; diff --git a/schema/crdb/8.0.0/up08.sql b/schema/crdb/8.0.0/up08.sql new file mode 100644 index 0000000000..c84480feba --- /dev/null +++ b/schema/crdb/8.0.0/up08.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS hold_time INT8; diff --git a/schema/crdb/8.0.0/up09.sql b/schema/crdb/8.0.0/up09.sql new file mode 100644 index 0000000000..82f645c753 --- /dev/null +++ b/schema/crdb/8.0.0/up09.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS idle_hold_time INT8; diff --git a/schema/crdb/8.0.0/up10.sql b/schema/crdb/8.0.0/up10.sql new file mode 100644 index 0000000000..a672953991 --- /dev/null +++ b/schema/crdb/8.0.0/up10.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS delay_open INT8; diff --git a/schema/crdb/8.0.0/up11.sql b/schema/crdb/8.0.0/up11.sql new file mode 100644 index 0000000000..63f16a011f --- /dev/null +++ b/schema/crdb/8.0.0/up11.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS connect_retry INT8; diff --git a/schema/crdb/8.0.0/up12.sql b/schema/crdb/8.0.0/up12.sql new file mode 100644 index 0000000000..431d10cd3c --- /dev/null +++ b/schema/crdb/8.0.0/up12.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS keepalive INT8; diff --git a/schema/crdb/8.0.0/up13.sql b/schema/crdb/8.0.0/up13.sql new file mode 100644 index 0000000000..44bfd90b8c --- /dev/null +++ b/schema/crdb/8.0.0/up13.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.rack ADD COLUMN IF NOT EXISTS rack_subnet INET; diff --git a/schema/crdb/8.0.0/up14.sql b/schema/crdb/8.0.0/up14.sql new file mode 100644 index 0000000000..18ce39e61c --- /dev/null +++ b/schema/crdb/8.0.0/up14.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS omicron.public.bootstore_keys ( + key TEXT NOT NULL PRIMARY KEY, + generation INT8 NOT NULL +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 4d0589b3a0..0fdaf5083c 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -63,7 +63,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.rack ( initialized BOOL NOT NULL, /* Used to configure the updates service URL */ - tuf_base_url STRING(512) + tuf_base_url STRING(512), + + /* The IPv6 underlay /56 prefix for the rack */ + rack_subnet INET ); /* @@ -198,7 +201,8 @@ CREATE TYPE IF NOT EXISTS omicron.public.service_kind AS ENUM ( 'nexus', 'ntp', 'oximeter', - 'tfport' + 'tfport', + 'mgd' ); CREATE TABLE IF NOT EXISTS omicron.public.service ( @@ -2441,10 +2445,14 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_route_config ( CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_bgp_peer_config ( port_settings_id UUID, - bgp_announce_set_id UUID NOT NULL, bgp_config_id UUID NOT NULL, interface_name TEXT, addr INET, + hold_time INT8, + idle_hold_time INT8, + delay_open INT8, + connect_retry INT8, + keepalive INT8, /* TODO https://github.com/oxidecomputer/omicron/issues/3013 */ PRIMARY KEY (port_settings_id, interface_name, addr) @@ -2458,7 +2466,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.bgp_config ( time_modified TIMESTAMPTZ NOT NULL, time_deleted TIMESTAMPTZ, asn INT8 NOT NULL, - vrf TEXT + vrf TEXT, + bgp_announce_set_id UUID NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS lookup_bgp_config_by_name ON omicron.public.bgp_config ( @@ -2500,6 +2509,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_address_config ( PRIMARY KEY (port_settings_id, address, interface_name) ); +CREATE TABLE IF NOT EXISTS omicron.public.bootstore_keys ( + key TEXT NOT NULL PRIMARY KEY, + generation INT8 NOT NULL +); + /* * The `sled_instance` view's definition needs to be modified in a separate * transaction from the transaction that created it. @@ -2559,6 +2573,27 @@ FROM WHERE instance.time_deleted IS NULL AND vmm.time_deleted IS NULL; +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM ( + 'Firecode', + 'None', + 'Rs' +); + +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( + '0G', + '1G', + '10G', + '25G', + '40G', + '50G', + '100G', + '200G', + '400G' +); + +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; + CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( -- There should only be one row of this table for the whole DB. -- It's a little goofy, but filter on "singleton = true" before querying @@ -2585,7 +2620,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '7.0.0', NULL) + ( TRUE, NOW(), NOW(), '8.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 4a8b02d23d..39a9a68acc 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -91,6 +91,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "required": [ + "asn", + "originate" + ], + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/definitions/Ipv4Network" + } + } + } + }, + "BgpPeerConfig": { + "type": "object", + "required": [ + "addr", + "asn", + "port" + ], + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + } + }, "BootstrapAddressDiscovery": { "oneOf": [ { @@ -149,6 +196,27 @@ } } }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/definitions/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/definitions/Ipv6Network" + } + ] + } + ], + "x-rust-type": "ipnetwork::IpNetwork" + }, "IpRange": { "oneOf": [ { @@ -201,6 +269,11 @@ "type": "string", "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$", + "x-rust-type": "ipnetwork::Ipv6Network" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -244,6 +317,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ], + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/definitions/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/definitions/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/definitions/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/definitions/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/definitions/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/definitions/PortSpeed" + } + ] + } + } + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -336,7 +472,7 @@ "description": "Initial rack network configuration", "anyOf": [ { - "$ref": "#/definitions/RackNetworkConfig" + "$ref": "#/definitions/RackNetworkConfigV1" }, { "type": "null" @@ -367,15 +503,24 @@ } } }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ], "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/definitions/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -386,12 +531,15 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/definitions/UplinkConfig" + "$ref": "#/definitions/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/definitions/Ipv6Network" } } }, @@ -414,6 +562,28 @@ } } }, + "RouteConfig": { + "type": "object", + "required": [ + "destination", + "nexthop" + ], + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/definitions/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + } + }, "StartSledAgentRequest": { "description": "Configuration information for launching a Sled Agent.", "type": "object", @@ -484,69 +654,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ], - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/definitions/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/definitions/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/definitions/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/definitions/PortSpeed" - } - ] - }, - "uplink_vid": { - "description": "VLAN id to use for uplink", - "type": [ - "integer", - "null" - ], - "format": "uint16", - "minimum": 0.0 - } - } - }, "UserId": { "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", "type": "string" diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 636c9665ef..ff9644773a 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -55,7 +55,7 @@ reqwest = { workspace = true, features = ["rustls-tls", "stream"] } schemars = { workspace = true, features = [ "chrono", "uuid1" ] } semver.workspace = true serde.workspace = true -serde_json.workspace = true +serde_json = {workspace = true, features = ["raw_value"]} sha3.workspace = true sled-agent-client.workspace = true sled-hardware.workspace = true diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 61d4c84af3..6c19080e9c 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -7,29 +7,31 @@ use anyhow::{anyhow, Context}; use bootstore::schemes::v0 as bootstore; use ddm_admin_client::{Client as DdmAdminClient, DdmError}; -use dpd_client::types::Ipv6Entry; +use dpd_client::types::{Ipv6Entry, RouteSettingsV6}; use dpd_client::types::{ LinkCreate, LinkId, LinkSettings, PortId, PortSettings, RouteSettingsV4, }; use dpd_client::Client as DpdClient; -use dpd_client::Ipv4Cidr; use futures::future; use gateway_client::Client as MgsClient; use internal_dns::resolver::{ResolveError, Resolver as DnsResolver}; use internal_dns::ServiceName; -use omicron_common::address::{Ipv6Subnet, AZ_PREFIX, MGS_PORT}; +use ipnetwork::{IpNetwork, Ipv6Network}; +use omicron_common::address::{Ipv6Subnet, MGS_PORT}; use omicron_common::address::{DDMD_PORT, DENDRITE_PORT}; use omicron_common::api::internal::shared::{ - PortFec, PortSpeed, RackNetworkConfig, SwitchLocation, UplinkConfig, + PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, RackNetworkConfigV1, + SwitchLocation, UplinkConfig, }; use omicron_common::backoff::{ retry_notify, retry_policy_local, BackoffError, ExponentialBackoff, ExponentialBackoffBuilder, }; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use slog::Logger; use std::collections::{HashMap, HashSet}; -use std::net::{IpAddr, Ipv6Addr, SocketAddrV6}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6}; use std::time::{Duration, Instant}; use thiserror::Error; @@ -107,11 +109,11 @@ impl<'a> EarlyNetworkSetup<'a> { resolver: &DnsResolver, config: &RackNetworkConfig, ) -> HashSet { - // Which switches have uplinks? + // Which switches have configured ports? let uplinked_switches = config - .uplinks + .ports .iter() - .map(|uplink_config| uplink_config.switch) + .map(|port_config| port_config.switch) .collect::>(); // If we have no uplinks, we have nothing to look up. @@ -342,7 +344,7 @@ impl<'a> EarlyNetworkSetup<'a> { &mut self, rack_network_config: &RackNetworkConfig, switch_zone_underlay_ip: Ipv6Addr, - ) -> Result, EarlyNetworkSetupError> { + ) -> Result, EarlyNetworkSetupError> { // First, we have to know which switch we are: ask MGS. info!( self.log, @@ -385,10 +387,10 @@ impl<'a> EarlyNetworkSetup<'a> { }; // We now know which switch we are: filter the uplinks to just ours. - let our_uplinks = rack_network_config - .uplinks + let our_ports = rack_network_config + .ports .iter() - .filter(|uplink| uplink.switch == switch_location) + .filter(|port| port.switch == switch_location) .cloned() .collect::>(); @@ -396,7 +398,7 @@ impl<'a> EarlyNetworkSetup<'a> { self.log, "Initializing {} Uplinks on {switch_location:?} at \ {switch_zone_underlay_ip}", - our_uplinks.len(), + our_ports.len(), ); let dpd = DpdClient::new( &format!("http://[{}]:{}", switch_zone_underlay_ip, DENDRITE_PORT), @@ -408,9 +410,9 @@ impl<'a> EarlyNetworkSetup<'a> { // configure uplink for each requested uplink in configuration that // matches our switch_location - for uplink_config in &our_uplinks { + for port_config in &our_ports { let (ipv6_entry, dpd_port_settings, port_id) = - self.build_uplink_config(uplink_config)?; + self.build_port_config(port_config)?; self.wait_for_dendrite(&dpd).await; @@ -446,14 +448,14 @@ impl<'a> EarlyNetworkSetup<'a> { ddmd_client.advertise_prefix(Ipv6Subnet::new(ipv6_entry.addr)); } - Ok(our_uplinks) + Ok(our_ports) } - fn build_uplink_config( + fn build_port_config( &self, - uplink_config: &UplinkConfig, + port_config: &PortConfigV1, ) -> Result<(Ipv6Entry, PortSettings, PortId), EarlyNetworkSetupError> { - info!(self.log, "Building Uplink Configuration"); + info!(self.log, "Building Port Configuration"); let ipv6_entry = Ipv6Entry { addr: BOUNDARY_SERVICES_ADDR.parse().map_err(|e| { EarlyNetworkSetupError::BadConfig(format!( @@ -469,41 +471,57 @@ impl<'a> EarlyNetworkSetup<'a> { v6_routes: HashMap::new(), }; let link_id = LinkId(0); + + let mut addrs = Vec::new(); + for a in &port_config.addresses { + addrs.push(a.ip()); + } + // TODO We're discarding the `uplink_cidr.prefix()` here and only using // the IP address; at some point we probably need to give the full CIDR // to dendrite? - let addr = IpAddr::V4(uplink_config.uplink_cidr.ip()); let link_settings = LinkSettings { // TODO Allow user to configure link properties // https://github.com/oxidecomputer/omicron/issues/3061 params: LinkCreate { autoneg: false, kr: false, - fec: convert_fec(&uplink_config.uplink_port_fec), - speed: convert_speed(&uplink_config.uplink_port_speed), + fec: convert_fec(&port_config.uplink_port_fec), + speed: convert_speed(&port_config.uplink_port_speed), }, - addrs: vec![addr], + //addrs: vec![addr], + addrs, }; dpd_port_settings.links.insert(link_id.to_string(), link_settings); - let port_id: PortId = - uplink_config.uplink_port.parse().map_err(|e| { - EarlyNetworkSetupError::BadConfig(format!( - concat!( - "could not use value provided to", - "rack_network_config.uplink_port as PortID: {}" - ), - e - )) - })?; - dpd_port_settings.v4_routes.insert( - Ipv4Cidr { prefix: "0.0.0.0".parse().unwrap(), prefix_len: 0 } - .to_string(), - RouteSettingsV4 { - link_id: link_id.0, - vid: uplink_config.uplink_vid, - nexthop: uplink_config.gateway_ip, - }, - ); + let port_id: PortId = port_config.port.parse().map_err(|e| { + EarlyNetworkSetupError::BadConfig(format!( + concat!( + "could not use value provided to", + "rack_network_config.uplink_port as PortID: {}" + ), + e + )) + })?; + + for r in &port_config.routes { + if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) = + (r.destination, r.nexthop) + { + dpd_port_settings.v4_routes.insert( + dst.to_string(), + RouteSettingsV4 { link_id: link_id.0, nexthop, vid: None }, + ); + } + if let (IpNetwork::V6(dst), IpAddr::V6(nexthop)) = + (r.destination, r.nexthop) + { + dpd_port_settings.v6_routes.insert( + dst.to_string(), + RouteSettingsV6 { link_id: link_id.0, nexthop, vid: None }, + ); + } + } + Ok((ipv6_entry, dpd_port_settings, port_id)) } @@ -546,33 +564,68 @@ fn retry_policy_switch_mapping() -> ExponentialBackoff { .build() } +// The first production version of the `EarlyNetworkConfig`. +// +// If this version is in the bootstore than we need to convert it to +// `EarlyNetworkConfigV1`. +// +// Once we do this for all customers that have initialized racks with the +// old version we can go ahead and remove this type and its conversion code +// altogether. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +struct EarlyNetworkConfigV0 { + // The current generation number of data as stored in CRDB. + // The initial generation is set during RSS time and then only mutated + // by Nexus. + pub generation: u64, + + pub rack_subnet: Ipv6Addr, + + /// The external NTP server addresses. + pub ntp_servers: Vec, + + // Rack network configuration as delivered from RSS and only existing at + // generation 1 + pub rack_network_config: Option, +} + /// Network configuration required to bring up the control plane /// /// The fields in this structure are those from /// [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This /// is just for the initial rack configuration and cold boot purposes. Updates -/// will come from Nexus in the future. -#[derive(Clone, Debug, Deserialize, Serialize)] +/// come from Nexus. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] pub struct EarlyNetworkConfig { - // The version of data. Always `1` when created from RSS. + // The current generation number of data as stored in CRDB. + // The initial generation is set during RSS time and then only mutated + // by Nexus. pub generation: u64, - pub rack_subnet: Ipv6Addr, + // Which version of the data structure do we have. This is to help with + // deserialization and conversion in future updates. + pub schema_version: u32, + + // The actual configuration details + pub body: EarlyNetworkConfigBody, +} +/// This is the actual configuration of EarlyNetworking. +/// +/// We nest it below the "header" of `generation` and `schema_version` so that +/// we can perform partial deserialization of `EarlyNetworkConfig` to only read +/// the header and defer deserialization of the body once we know the schema +/// version. This is possible via the use of [`serde_json::value::RawValue`] in +/// future (post-v1) deserialization paths. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct EarlyNetworkConfigBody { /// The external NTP server addresses. pub ntp_servers: Vec, - /// A copy of the initial rack network configuration when we are in - /// generation `1`. + // Rack network configuration as delivered from RSS or Nexus pub rack_network_config: Option, } -impl EarlyNetworkConfig { - pub fn az_subnet(&self) -> Ipv6Subnet { - Ipv6Subnet::::new(self.rack_subnet) - } -} - impl From for bootstore::NetworkConfig { fn from(value: EarlyNetworkConfig) -> Self { // Can this ever actually fail? @@ -586,13 +639,77 @@ impl From for bootstore::NetworkConfig { } } +// Note: This currently only converts between v0 and v1 or deserializes v1 of +// `EarlyNetworkConfig`. impl TryFrom for EarlyNetworkConfig { type Error = serde_json::Error; fn try_from( value: bootstore::NetworkConfig, ) -> std::result::Result { - serde_json::from_slice(&value.blob) + // Try to deserialize the latest version of the data structure (v1). If + // that succeeds we are done. + if let Ok(val) = + serde_json::from_slice::(&value.blob) + { + return Ok(val); + } + + // We don't have the latest version. Try to deserialize v0 and then + // convert it to the latest version. + let v0 = serde_json::from_slice::(&value.blob)?; + + Ok(EarlyNetworkConfig { + generation: v0.generation, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: v0.ntp_servers, + rack_network_config: v0.rack_network_config.map(|v0_config| { + RackNetworkConfigV0::to_v1(v0.rack_subnet, v0_config) + }), + }, + }) + } +} + +/// Deprecated, use `RackNetworkConfig` instead. Cannot actually deprecate due to +/// +/// +/// Our first version of `RackNetworkConfig`. If this exists in the bootstore, we +/// upgrade out of it into `RackNetworkConfigV1` or later versions if possible. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct RackNetworkConfigV0 { + // TODO: #3591 Consider making infra-ip ranges implicit for uplinks + /// First ip address to be used for configuring network infrastructure + pub infra_ip_first: Ipv4Addr, + /// Last ip address to be used for configuring network infrastructure + pub infra_ip_last: Ipv4Addr, + /// Uplinks for connecting the rack to external networks + pub uplinks: Vec, +} + +impl RackNetworkConfigV0 { + /// Convert from `RackNetworkConfigV0` to `RackNetworkConfigV1` + /// + /// We cannot use `From for `RackNetworkConfigV1` + /// because the `rack_subnet` field does not exist in `RackNetworkConfigV0` + /// and must be passed in from the `EarlyNetworkConfigV0` struct which + /// contains the `RackNetworkConfivV0` struct. + pub fn to_v1( + rack_subnet: Ipv6Addr, + v0: RackNetworkConfigV0, + ) -> RackNetworkConfigV1 { + RackNetworkConfigV1 { + rack_subnet: Ipv6Network::new(rack_subnet, 56).unwrap(), + infra_ip_first: v0.infra_ip_first, + infra_ip_last: v0.infra_ip_last, + ports: v0 + .uplinks + .into_iter() + .map(|uplink| PortConfigV1::from(uplink)) + .collect(), + bgp: vec![], + } } } @@ -621,3 +738,66 @@ fn convert_fec(fec: &PortFec) -> dpd_client::types::PortFec { PortFec::Rs => dpd_client::types::PortFec::Rs, } } + +#[cfg(test)] +mod tests { + use super::*; + use omicron_common::api::internal::shared::RouteConfig; + + #[test] + fn serialized_early_network_config_v0_to_v1_conversion() { + let v0 = EarlyNetworkConfigV0 { + generation: 1, + rack_subnet: Ipv6Addr::UNSPECIFIED, + ntp_servers: Vec::new(), + rack_network_config: Some(RackNetworkConfigV0 { + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + uplinks: vec![UplinkConfig { + gateway_ip: Ipv4Addr::UNSPECIFIED, + switch: SwitchLocation::Switch0, + uplink_port: "Port0".to_string(), + uplink_port_speed: PortSpeed::Speed100G, + uplink_port_fec: PortFec::None, + uplink_cidr: "192.168.0.1/16".parse().unwrap(), + uplink_vid: None, + }], + }), + }; + + let v0_serialized = serde_json::to_vec(&v0).unwrap(); + let bootstore_conf = + bootstore::NetworkConfig { generation: 1, blob: v0_serialized }; + + let v1 = EarlyNetworkConfig::try_from(bootstore_conf).unwrap(); + let v0_rack_network_config = v0.rack_network_config.unwrap(); + let uplink = v0_rack_network_config.uplinks[0].clone(); + let expected = EarlyNetworkConfig { + generation: 1, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: v0.ntp_servers.clone(), + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: Ipv6Network::new(v0.rack_subnet, 56).unwrap(), + infra_ip_first: v0_rack_network_config.infra_ip_first, + infra_ip_last: v0_rack_network_config.infra_ip_last, + ports: vec![PortConfigV1 { + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: uplink.gateway_ip.into(), + }], + addresses: vec![uplink.uplink_cidr.into()], + switch: uplink.switch, + port: uplink.uplink_port, + uplink_port_speed: uplink.uplink_port_speed, + uplink_port_fec: uplink.uplink_port_fec, + bgp_peers: vec![], + }], + bgp: vec![], + }), + }, + }; + + assert_eq!(expected, v1); + } +} diff --git a/sled-agent/src/bootstrap/maghemite.rs b/sled-agent/src/bootstrap/maghemite.rs index 1adc677b23..2cf0eaf190 100644 --- a/sled-agent/src/bootstrap/maghemite.rs +++ b/sled-agent/src/bootstrap/maghemite.rs @@ -8,7 +8,7 @@ use illumos_utils::addrobj::AddrObject; use slog::Logger; use thiserror::Error; -const SERVICE_FMRI: &str = "svc:/system/illumos/mg-ddm"; +const SERVICE_FMRI: &str = "svc:/oxide/mg-ddm"; const MANIFEST_PATH: &str = "/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"; #[derive(Debug, Error)] diff --git a/sled-agent/src/bootstrap/secret_retriever.rs b/sled-agent/src/bootstrap/secret_retriever.rs index 5cae06310c..1d5ac10ac5 100644 --- a/sled-agent/src/bootstrap/secret_retriever.rs +++ b/sled-agent/src/bootstrap/secret_retriever.rs @@ -14,9 +14,9 @@ use std::sync::OnceLock; static MAYBE_LRTQ_RETRIEVER: OnceLock = OnceLock::new(); -/// A [`key-manager::SecretRetriever`] that either uses a -/// [`LocalSecretRetriever`] or [`LrtqSecretRetriever`] under the hood depending -/// upon how many sleds are in the cluster at rack init time. +/// A [`key_manager::SecretRetriever`] that either uses a +/// [`HardcodedSecretRetriever`] or [`LrtqSecretRetriever`] under the +/// hood depending upon how many sleds are in the cluster at rack init time. pub struct LrtqOrHardcodedSecretRetriever {} impl LrtqOrHardcodedSecretRetriever { diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index 0cbbf0678b..9ed3ad582d 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -528,7 +528,7 @@ fn start_dropshot_server( /// /// TODO-correctness Subsequent steps may assume all M.2s that will ever be /// present are present once we return from this function; see -/// https://github.com/oxidecomputer/omicron/issues/3815. +/// . async fn wait_for_boot_m2(storage_resources: &StorageResources, log: &Logger) { // Wait for at least the M.2 we booted from to show up. loop { diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 2ab8273e39..68330d0c0e 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -5,6 +5,7 @@ //! HTTP entrypoint functions for the sled agent's exposed API use super::sled_agent::SledAgent; +use crate::bootstrap::early_networking::EarlyNetworkConfig; use crate::params::{ CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, @@ -14,6 +15,7 @@ use crate::params::{ }; use crate::sled_agent::Error as SledAgentError; use crate::zone_bundle; +use bootstore::schemes::v0::NetworkConfig; use camino::Utf8PathBuf; use dropshot::{ endpoint, ApiDescription, FreeformBody, HttpError, HttpResponseCreated, @@ -24,9 +26,10 @@ use illumos_utils::opte::params::{ DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost, }; use omicron_common::api::external::Error; -use omicron_common::api::internal::nexus::DiskRuntimeState; -use omicron_common::api::internal::nexus::SledInstanceState; -use omicron_common::api::internal::nexus::UpdateArtifactId; +use omicron_common::api::internal::nexus::{ + DiskRuntimeState, SledInstanceState, UpdateArtifactId, +}; +use omicron_common::api::internal::shared::SwitchPorts; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -62,6 +65,9 @@ pub fn api() -> SledApiDescription { api.register(update_artifact)?; api.register(vpc_firewall_rules_put)?; api.register(zpools_get)?; + api.register(uplink_ensure)?; + api.register(read_network_bootstore_config_cache)?; + api.register(write_network_bootstore_config)?; Ok(()) } @@ -630,3 +636,73 @@ async fn timesync_get( let sa = rqctx.context(); Ok(HttpResponseOk(sa.timesync_get().await.map_err(|e| Error::from(e))?)) } + +#[endpoint { + method = POST, + path = "/switch-ports", +}] +async fn uplink_ensure( + rqctx: RequestContext, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + sa.ensure_scrimlet_host_ports(body.into_inner().uplinks).await?; + Ok(HttpResponseUpdatedNoContent()) +} + +/// This API endpoint is only reading the local sled agent's view of the +/// bootstore. The boostore is a distributed data store that is eventually +/// consistent. Reads from individual nodes may not represent the latest state. +#[endpoint { + method = GET, + path = "/network-bootstore-config", +}] +async fn read_network_bootstore_config_cache( + rqctx: RequestContext, +) -> Result, HttpError> { + let sa = rqctx.context(); + let bs = sa.bootstore(); + + let config = bs.get_network_config().await.map_err(|e| { + HttpError::for_internal_error(format!("failed to get bootstore: {e}")) + })?; + + let config = match config { + Some(config) => EarlyNetworkConfig::try_from(config).map_err(|e| { + HttpError::for_internal_error(format!( + "deserialize early network config: {e}" + )) + })?, + None => { + return Err(HttpError::for_unavail( + None, + "early network config does not exist yet".into(), + )); + } + }; + + Ok(HttpResponseOk(config)) +} + +#[endpoint { + method = PUT, + path = "/network-bootstore-config", +}] +async fn write_network_bootstore_config( + rqctx: RequestContext, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + let bs = sa.bootstore(); + let config = body.into_inner(); + + bs.update_network_config(NetworkConfig::from(config)).await.map_err( + |e| { + HttpError::for_internal_error(format!( + "failed to write updated config to boot store: {e}" + )) + }, + )?; + + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index e1c8b05cde..5fda3c1ae6 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -351,10 +351,12 @@ pub enum ServiceType { #[serde(skip)] Uplink, #[serde(skip)] - Maghemite { + MgDdm { mode: String, }, #[serde(skip)] + Mgd, + #[serde(skip)] SpSim, CruciblePantry { address: SocketAddrV6, @@ -404,7 +406,8 @@ impl std::fmt::Display for ServiceType { ServiceType::CruciblePantry { .. } => write!(f, "crucible/pantry"), ServiceType::BoundaryNtp { .. } | ServiceType::InternalNtp { .. } => write!(f, "ntp"), - ServiceType::Maghemite { .. } => write!(f, "mg-ddm"), + ServiceType::MgDdm { .. } => write!(f, "mg-ddm"), + ServiceType::Mgd => write!(f, "mgd"), ServiceType::SpSim => write!(f, "sp-sim"), ServiceType::Clickhouse { .. } => write!(f, "clickhouse"), ServiceType::ClickhouseKeeper { .. } => { @@ -421,13 +424,7 @@ impl crate::smf_helper::Service for ServiceType { self.to_string() } fn smf_name(&self) -> String { - match self { - // NOTE: This style of service-naming is deprecated - ServiceType::Maghemite { .. } => { - format!("svc:/system/illumos/{}", self.service_name()) - } - _ => format!("svc:/oxide/{}", self.service_name()), - } + format!("svc:/oxide/{}", self.service_name()) } fn should_import(&self) -> bool { true @@ -527,7 +524,8 @@ impl TryFrom for sled_agent_client::types::ServiceType { | St::Dendrite { .. } | St::Tfport { .. } | St::Uplink - | St::Maghemite { .. } => Err(AutonomousServiceOnlyError), + | St::Mgd + | St::MgDdm { .. } => Err(AutonomousServiceOnlyError), } } } @@ -826,7 +824,8 @@ impl ServiceZoneRequest { | ServiceType::SpSim | ServiceType::Wicketd { .. } | ServiceType::Dendrite { .. } - | ServiceType::Maghemite { .. } + | ServiceType::MgDdm { .. } + | ServiceType::Mgd | ServiceType::Tfport { .. } | ServiceType::Uplink => { return Err(AutonomousServiceOnlyError); diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 2183aa7b63..3dac5d7d1e 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -19,10 +19,11 @@ use internal_dns::{ServiceName, DNS_ZONE}; use omicron_common::address::{ get_sled_address, get_switch_zone_address, Ipv6Subnet, ReservedRackSubnet, DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, DNS_REDUNDANCY, MAX_DNS_REDUNDANCY, - MGS_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, RSS_RESERVED_ADDRESSES, + MGD_PORT, MGS_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, RSS_RESERVED_ADDRESSES, SLED_PREFIX, }; use omicron_common::api::external::{MacAddr, Vni}; +use omicron_common::api::internal::shared::SwitchLocation; use omicron_common::api::internal::shared::{ NetworkInterface, NetworkInterfaceKind, SourceNatConfig, }; @@ -276,7 +277,7 @@ impl Plan { "No scrimlets observed".to_string(), )); } - for sled in scrimlets { + for (i, sled) in scrimlets.iter().enumerate() { let address = get_switch_zone_address(sled.subnet); let zone = dns_builder.host_dendrite(sled.sled_id, address).unwrap(); @@ -294,6 +295,18 @@ impl Plan { MGS_PORT, ) .unwrap(); + dns_builder + .service_backend_zone(ServiceName::Mgd, &zone, MGD_PORT) + .unwrap(); + + // TODO only works for single rack + let sled_address = get_sled_address(sled.subnet); + let switch_location = if i == 0 { + SwitchLocation::Switch0 + } else { + SwitchLocation::Switch1 + }; + dns_builder.host_scrimlet(switch_location, sled_address).unwrap(); } // We'll stripe most services across all available Sleds, round-robin diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 805c889295..7f6469d2c0 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -57,7 +57,8 @@ use super::config::SetupServiceConfig as Config; use crate::bootstrap::config::BOOTSTRAP_AGENT_HTTP_PORT; use crate::bootstrap::early_networking::{ - EarlyNetworkConfig, EarlyNetworkSetup, EarlyNetworkSetupError, + EarlyNetworkConfig, EarlyNetworkConfigBody, EarlyNetworkSetup, + EarlyNetworkSetupError, }; use crate::bootstrap::params::BootstrapAddressDiscovery; use crate::bootstrap::params::StartSledAgentRequest; @@ -575,17 +576,25 @@ impl ServiceInner { let rack_network_config = match &config.rack_network_config { Some(config) => { - let value = NexusTypes::RackNetworkConfig { + let value = NexusTypes::RackNetworkConfigV1 { + rack_subnet: config.rack_subnet, infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, - uplinks: config - .uplinks + ports: config + .ports .iter() - .map(|config| NexusTypes::UplinkConfig { - gateway_ip: config.gateway_ip, + .map(|config| NexusTypes::PortConfigV1 { + port: config.port.clone(), + routes: config + .routes + .iter() + .map(|r| NexusTypes::RouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), switch: config.switch.into(), - uplink_cidr: config.uplink_cidr, - uplink_port: config.uplink_port.clone(), uplink_port_speed: config .uplink_port_speed .clone() @@ -594,7 +603,23 @@ impl ServiceInner { .uplink_port_fec .clone() .into(), - uplink_vid: config.uplink_vid, + bgp_peers: config + .bgp_peers + .iter() + .map(|b| NexusTypes::BgpPeerConfig { + addr: b.addr, + asn: b.asn, + port: b.port.clone(), + }) + .collect(), + }) + .collect(), + bgp: config + .bgp + .iter() + .map(|config| NexusTypes::BgpConfig { + asn: config.asn, + originate: config.originate.clone(), }) .collect(), }; @@ -872,9 +897,11 @@ impl ServiceInner { // from the bootstore". let early_network_config = EarlyNetworkConfig { generation: 1, - rack_subnet: config.rack_subnet, - ntp_servers: config.ntp_servers.clone(), - rack_network_config: config.rack_network_config.clone(), + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: config.ntp_servers.clone(), + rack_network_config: config.rack_network_config.clone(), + }, }; info!(self.log, "Writing Rack Network Configuration to bootstore"); bootstore.update_network_config(early_network_config.into()).await?; diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 06d3ae1977..a9be0e7c4a 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -77,7 +77,9 @@ use omicron_common::address::WICKETD_NEXUS_PROXY_PORT; use omicron_common::address::WICKETD_PORT; use omicron_common::address::{Ipv6Subnet, NEXUS_TECHPORT_EXTERNAL_PORT}; use omicron_common::api::external::Generation; -use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::{ + HostPortConfig, RackNetworkConfig, +}; use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, retry_policy_local, BackoffError, @@ -96,8 +98,8 @@ use sled_hardware::underlay::BOOTSTRAP_PREFIX; use sled_hardware::Baseboard; use sled_hardware::SledMode; use slog::Logger; +use std::collections::BTreeMap; use std::collections::HashSet; -use std::collections::{BTreeMap, HashMap}; use std::iter; use std::iter::FromIterator; use std::net::{IpAddr, Ipv6Addr, SocketAddr}; @@ -758,7 +760,7 @@ impl ServiceManager { } } } - ServiceType::Maghemite { .. } => { + ServiceType::MgDdm { .. } => { // If on a non-gimlet, sled-agent can be configured to map // links into the switch zone. Validate those links here. for link in &self.inner.switch_zone_maghemite_links { @@ -1953,8 +1955,13 @@ impl ServiceManager { // Nothing to do here - this service is special and // configured in `ensure_switch_zone_uplinks_configured` } - ServiceType::Maghemite { mode } => { - info!(self.inner.log, "Setting up Maghemite service"); + ServiceType::Mgd => { + info!(self.inner.log, "Setting up mgd service"); + smfh.setprop("config/admin_host", "::")?; + smfh.refresh()?; + } + ServiceType::MgDdm { mode } => { + info!(self.inner.log, "Setting up mg-ddm service"); smfh.setprop("config/mode", &mode)?; smfh.setprop("config/admin_host", "::")?; @@ -2015,8 +2022,8 @@ impl ServiceManager { )?; if is_gimlet { - // Maghemite for a scrimlet needs to be configured to - // talk to dendrite + // Ddm for a scrimlet needs to be configured to talk to + // dendrite smfh.setprop("config/dpd_host", "[::1]")?; smfh.setprop("config/dpd_port", DENDRITE_PORT)?; } @@ -2505,7 +2512,8 @@ impl ServiceManager { ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, ServiceType::Uplink, ServiceType::Wicketd { baseboard }, - ServiceType::Maghemite { mode: "transit".to_string() }, + ServiceType::Mgd, + ServiceType::MgDdm { mode: "transit".to_string() }, ] } @@ -2528,7 +2536,8 @@ impl ServiceManager { ServiceType::ManagementGatewayService, ServiceType::Uplink, ServiceType::Wicketd { baseboard }, - ServiceType::Maghemite { mode: "transit".to_string() }, + ServiceType::Mgd, + ServiceType::MgDdm { mode: "transit".to_string() }, ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, ServiceType::SpSim, ] @@ -2583,10 +2592,20 @@ impl ServiceManager { let log = &self.inner.log; // Configure uplinks via DPD in our switch zone. - let our_uplinks = EarlyNetworkSetup::new(log) + let our_ports = EarlyNetworkSetup::new(log) .init_switch_config(rack_network_config, switch_zone_ip) - .await?; + .await? + .into_iter() + .map(From::from) + .collect(); + self.ensure_scrimlet_host_ports(our_ports).await + } + + pub async fn ensure_scrimlet_host_ports( + &self, + our_ports: Vec, + ) -> Result<(), Error> { // We expect the switch zone to be running, as we're called immediately // after `ensure_zone()` above and we just successfully configured // uplinks via DPD running in our switch zone. If somehow we're in any @@ -2617,22 +2636,14 @@ impl ServiceManager { smfh.delpropgroup("uplinks")?; smfh.addpropgroup("uplinks", "application")?; - // When naming the uplink ports, we need to append `_0`, `_1`, etc., for - // each use of any given port. We use a hashmap of counters of port name - // -> number of uplinks to correctly supply that suffix. - let mut port_count = HashMap::new(); - for uplink_config in &our_uplinks { - let this_port_count: &mut usize = - port_count.entry(&uplink_config.uplink_port).or_insert(0); - smfh.addpropvalue_type( - &format!( - "uplinks/{}_{}", - uplink_config.uplink_port, *this_port_count - ), - &uplink_config.uplink_cidr.to_string(), - "astring", - )?; - *this_port_count += 1; + for port_config in &our_ports { + for addr in &port_config.addrs { + smfh.addpropvalue_type( + &format!("uplinks/{}_0", port_config.port,), + &addr.to_string(), + "astring", + )?; + } } smfh.refresh()?; @@ -2868,7 +2879,7 @@ impl ServiceManager { // Only configured in // `ensure_switch_zone_uplinks_configured` } - ServiceType::Maghemite { mode } => { + ServiceType::MgDdm { mode } => { smfh.delpropvalue("config/mode", "*")?; smfh.addpropvalue("config/mode", &mode)?; smfh.refresh()?; diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 08f6c7d10b..f77da11b0e 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -4,6 +4,9 @@ //! HTTP entrypoint functions for the sled agent's exposed API +use crate::bootstrap::early_networking::{ + EarlyNetworkConfig, EarlyNetworkConfigBody, +}; use crate::params::{ DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, InstancePutStateResponse, InstanceUnregisterResponse, @@ -19,11 +22,15 @@ use dropshot::RequestContext; use dropshot::TypedBody; use illumos_utils::opte::params::DeleteVirtualNetworkInterfaceHost; use illumos_utils::opte::params::SetVirtualNetworkInterfaceHost; +use ipnetwork::Ipv6Network; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::UpdateArtifactId; +use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::SwitchPorts; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::net::{Ipv4Addr, Ipv6Addr}; use std::sync::Arc; use uuid::Uuid; @@ -46,6 +53,9 @@ pub fn api() -> SledApiDescription { api.register(vpc_firewall_rules_put)?; api.register(set_v2p)?; api.register(del_v2p)?; + api.register(uplink_ensure)?; + api.register(read_network_bootstore_config)?; + api.register(write_network_bootstore_config)?; Ok(()) } @@ -327,3 +337,50 @@ async fn del_v2p( Ok(HttpResponseUpdatedNoContent()) } + +#[endpoint { + method = POST, + path = "/switch-ports", +}] +async fn uplink_ensure( + _rqctx: RequestContext>, + _body: TypedBody, +) -> Result { + Ok(HttpResponseUpdatedNoContent()) +} + +#[endpoint { + method = GET, + path = "/network-bootstore-config", +}] +async fn read_network_bootstore_config( + _rqctx: RequestContext>, +) -> Result, HttpError> { + let config = EarlyNetworkConfig { + generation: 0, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: Vec::new(), + rack_network_config: Some(RackNetworkConfig { + rack_subnet: Ipv6Network::new(Ipv6Addr::UNSPECIFIED, 56) + .unwrap(), + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports: Vec::new(), + bgp: Vec::new(), + }), + }, + }; + Ok(HttpResponseOk(config)) +} + +#[endpoint { + method = PUT, + path = "/network-bootstore-config", +}] +async fn write_network_bootstore_config( + _rqctx: RequestContext>, + _body: TypedBody, +) -> Result { + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 595d83a7ee..1f2fe8e1d8 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -94,7 +94,7 @@ impl Server { &config.id, &NexusTypes::SledAgentStartupInfo { sa_address: sa_address.to_string(), - role: NexusTypes::SledRole::Gimlet, + role: NexusTypes::SledRole::Scrimlet, baseboard: NexusTypes::Baseboard { serial_number: format!( "sim-{}", diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index b6f910220e..52513f081d 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -38,7 +38,9 @@ use omicron_common::api::external::Vni; use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; -use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::{ + HostPortConfig, RackNetworkConfig, +}; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, internal::nexus::UpdateArtifactId, @@ -237,6 +239,9 @@ struct SledAgentInner { // Object managing zone bundles. zone_bundler: zone_bundle::ZoneBundler, + + // A handle to the bootstore. + bootstore: bootstore::NodeHandle, } impl SledAgentInner { @@ -407,7 +412,7 @@ impl SledAgent { EarlyNetworkConfig::try_from(serialized_config) .map_err(|err| BackoffError::transient(err.to_string()))?; - Ok(early_network_config.rack_network_config) + Ok(early_network_config.body.rack_network_config) }; let rack_network_config: Option = retry_notify::<_, String, _, _, _, _>( @@ -458,6 +463,7 @@ impl SledAgent { nexus_request_queue: NexusRequestQueue::new(), rack_network_config, zone_bundler, + bootstore: bootstore.clone(), }), log: log.clone(), }; @@ -769,7 +775,7 @@ impl SledAgent { /// Idempotently ensures that a given instance is registered with this sled, /// i.e., that it can be addressed by future calls to - /// [`instance_ensure_state`]. + /// [`Self::instance_ensure_state`]. pub async fn instance_ensure_registered( &self, instance_id: Uuid, @@ -918,4 +924,19 @@ impl SledAgent { pub async fn timesync_get(&self) -> Result { self.inner.services.timesync_get().await.map_err(Error::from) } + + pub async fn ensure_scrimlet_host_ports( + &self, + uplinks: Vec, + ) -> Result<(), Error> { + self.inner + .services + .ensure_scrimlet_host_ports(uplinks) + .await + .map_err(Error::from) + } + + pub fn bootstore(&self) -> bootstore::NodeHandle { + self.inner.bootstore.clone() + } } diff --git a/smf/sled-agent/gimlet-standalone/config-rss.toml b/smf/sled-agent/gimlet-standalone/config-rss.toml index c6fbab49de..29a7a79eba 100644 --- a/smf/sled-agent/gimlet-standalone/config-rss.toml +++ b/smf/sled-agent/gimlet-standalone/config-rss.toml @@ -88,27 +88,34 @@ last = "192.168.1.29" # Configuration to bring up Boundary Services and make Nexus reachable from the # outside. See docs/how-to-run.adoc for more on what to put here. [rack_network_config] +rack_subnet = "fd00:1122:3344:01::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. infra_ip_first = "192.168.1.30" infra_ip_last = "192.168.1.30" +# Configurations for BGP routers to run on the scrimlets. +bgp = [] + # You can configure multiple uplinks by repeating the following stanza -[[rack_network_config.uplinks]] -# The gateway IP for the rack's external network -gateway_ip = "192.168.1.199" +[[rack_network_config.ports]] +# Routes associated with this port. +routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] +# Addresses associated with this port. +addresses = ["192.168.1.30/32"] # Name of the uplink port. This should always be "qsfp0" when using softnpu. -uplink_port = "qsfp0" +port = "qsfp0" +# The speed of this port. uplink_port_speed = "40G" +# The forward error correction mode for this port. uplink_port_fec="none" -# For softnpu, an address within the "infra" block above that will be used for -# the softnpu uplink port. You can just pick the first address in that pool. -uplink_cidr = "192.168.1.30/32" # Switch to use for the uplink. For single-rack deployments this can be # "switch0" (upper slot) or "switch1" (lower slot). For single-node softnpu # and dendrite stub environments, use "switch0" switch = "switch0" +# Neighbors we expect to peer with over BGP on this port. +bgp_peers = [] # Configuration for the initial Silo, user, and password. # diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index 8a009dd687..fea3cfa5d8 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -88,27 +88,34 @@ last = "192.168.1.29" # Configuration to bring up Boundary Services and make Nexus reachable from the # outside. See docs/how-to-run.adoc for more on what to put here. [rack_network_config] +rack_subnet = "fd00:1122:3344:01::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. infra_ip_first = "192.168.1.30" infra_ip_last = "192.168.1.30" +# Configurations for BGP routers to run on the scrimlets. +bgp = [] + # You can configure multiple uplinks by repeating the following stanza -[[rack_network_config.uplinks]] -# The gateway IP for the rack's external network -gateway_ip = "192.168.1.199" +[[rack_network_config.ports]] +# Routes associated with this port. +routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] +# Addresses associated with this port. +addresses = ["192.168.1.30/32"] # Name of the uplink port. This should always be "qsfp0" when using softnpu. -uplink_port = "qsfp0" +port = "qsfp0" +# The speed of this port. uplink_port_speed = "40G" +# The forward error correction mode for this port. uplink_port_fec="none" -# For softnpu, an address within the "infra" block above that will be used for -# the softnpu uplink port. You can just pick the first address in that pool. -uplink_cidr = "192.168.1.30/32" # Switch to use for the uplink. For single-rack deployments this can be # "switch0" (upper slot) or "switch1" (lower slot). For single-node softnpu # and dendrite stub environments, use "switch0" switch = "switch0" +# Neighbors we expect to peer with over BGP on this port. +bgp_peers = [] # Configuration for the initial Silo, user, and password. # diff --git a/test-utils/src/dev/dendrite.rs b/test-utils/src/dev/dendrite.rs index 520bf12401..8938595aa2 100644 --- a/test-utils/src/dev/dendrite.rs +++ b/test-utils/src/dev/dendrite.rs @@ -19,7 +19,7 @@ use tokio::{ /// Specifies the amount of time we will wait for `dpd` to launch, /// which is currently confirmed by watching `dpd`'s log output /// for a message specifying the address and port `dpd` is listening on. -pub const DENDRITE_TIMEOUT: Duration = Duration::new(5, 0); +pub const DENDRITE_TIMEOUT: Duration = Duration::new(30, 0); /// Represents a running instance of the Dendrite dataplane daemon (dpd). pub struct DendriteInstance { diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs new file mode 100644 index 0000000000..fa1f353896 --- /dev/null +++ b/test-utils/src/dev/maghemite.rs @@ -0,0 +1,155 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tools for managing Maghemite during development + +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::Context; +use tempfile::TempDir; +use tokio::{ + fs::File, + io::{AsyncBufReadExt, BufReader}, + time::{sleep, Instant}, +}; + +/// Specifies the amount of time we will wait for `mgd` to launch, +/// which is currently confirmed by watching `mgd`'s log output +/// for a message specifying the address and port `mgd` is listening on. +pub const MGD_TIMEOUT: Duration = Duration::new(5, 0); + +pub struct MgdInstance { + /// Port number the mgd instance is listening on. This can be provided + /// manually, or dynamically determined if a value of 0 is provided. + pub port: u16, + /// Arguments provided to the `mgd` cli command. + pub args: Vec, + /// Child process spawned by running `mgd` + pub child: Option, + /// Temporary directory where logging output and other files generated by + /// `mgd` are stored. + pub data_dir: Option, +} + +impl MgdInstance { + pub async fn start(mut port: u16) -> Result { + let temp_dir = TempDir::new()?; + + let args = vec![ + "run".to_string(), + "--admin-addr".into(), + "::1".into(), + "--admin-port".into(), + port.to_string(), + "--no-bgp-dispatcher".into(), + "--data-dir".into(), + temp_dir.path().display().to_string(), + ]; + + let child = tokio::process::Command::new("mgd") + .args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::from(redirect_file(temp_dir.path(), "mgd_stdout")?)) + .stderr(Stdio::from(redirect_file(temp_dir.path(), "mgd_stderr")?)) + .spawn() + .with_context(|| { + format!("failed to spawn `mgd` (with args: {:?})", &args) + })?; + + let child = Some(child); + + let temp_dir = temp_dir.into_path(); + if port == 0 { + port = discover_port( + temp_dir.join("mgd_stdout").display().to_string(), + ) + .await + .with_context(|| { + format!( + "failed to discover mgd port from files in {}", + temp_dir.display() + ) + })?; + } + + Ok(Self { port, args, child, data_dir: Some(temp_dir) }) + } + + pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { + if let Some(mut child) = self.child.take() { + child.start_kill().context("Sending SIGKILL to child")?; + child.wait().await.context("waiting for child")?; + } + if let Some(dir) = self.data_dir.take() { + std::fs::remove_dir_all(&dir).with_context(|| { + format!("cleaning up temporary directory {}", dir.display()) + })?; + } + Ok(()) + } +} + +impl Drop for MgdInstance { + fn drop(&mut self) { + if self.child.is_some() || self.data_dir.is_some() { + eprintln!( + "WARN: dropped MgdInstance without cleaning it up first \ + (there may still be a child process running and a \ + temporary directory leaked)" + ); + if let Some(child) = self.child.as_mut() { + let _ = child.start_kill(); + } + if let Some(path) = self.data_dir.take() { + eprintln!( + "WARN: mgd temporary directory leaked: {}", + path.display() + ); + } + } + } +} + +fn redirect_file( + temp_dir_path: &Path, + label: &str, +) -> Result { + let out_path = temp_dir_path.join(label); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&out_path) + .with_context(|| format!("open \"{}\"", out_path.display())) +} + +async fn discover_port(logfile: String) -> Result { + let timeout = Instant::now() + MGD_TIMEOUT; + tokio::time::timeout_at(timeout, find_mgd_port_in_log(logfile)) + .await + .context("time out while discovering mgd port number")? +} + +async fn find_mgd_port_in_log(logfile: String) -> Result { + let re = regex::Regex::new(r#""local_addr":"\[::1\]:?([0-9]+)""#).unwrap(); + let reader = BufReader::new(File::open(logfile).await?); + let mut lines = reader.lines(); + loop { + match lines.next_line().await? { + Some(line) => { + if let Some(cap) = re.captures(&line) { + // unwrap on get(1) should be ok, since captures() returns + // `None` if there are no matches found + let port = cap.get(1).unwrap(); + let result = port.as_str().parse::()?; + return Ok(result); + } + } + None => { + sleep(Duration::from_millis(10)).await; + } + } + } +} diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index dbd66fe1f8..e29da9c51e 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -8,6 +8,7 @@ pub mod clickhouse; pub mod db; pub mod dendrite; +pub mod maghemite; pub mod poll; #[cfg(feature = "seed-gen")] pub mod seed; diff --git a/tools/build-global-zone-packages.sh b/tools/build-global-zone-packages.sh index 54af9d6327..fc1ab42ade 100755 --- a/tools/build-global-zone-packages.sh +++ b/tools/build-global-zone-packages.sh @@ -12,7 +12,7 @@ out_dir="$(readlink -f "${2:-"$tarball_src_dir"}")" # Make sure needed packages exist deps=( "$tarball_src_dir/omicron-sled-agent.tar" - "$tarball_src_dir/maghemite.tar" + "$tarball_src_dir/mg-ddm-gz.tar" "$tarball_src_dir/propolis-server.tar.gz" "$tarball_src_dir/overlay.tar.gz" ) @@ -46,7 +46,7 @@ cd - pkg_dir="$tmp_gz/root/opt/oxide/mg-ddm" mkdir -p "$pkg_dir" cd "$pkg_dir" -tar -xvfz "$tarball_src_dir/maghemite.tar" +tar -xvfz "$tarball_src_dir/mg-ddm-gz.tar" cd - # propolis should be bundled with this OS: Put the propolis-server zone image diff --git a/tools/build-trampoline-global-zone-packages.sh b/tools/build-trampoline-global-zone-packages.sh index 87013fb563..d8df0f8921 100755 --- a/tools/build-trampoline-global-zone-packages.sh +++ b/tools/build-trampoline-global-zone-packages.sh @@ -12,7 +12,7 @@ out_dir="$(readlink -f "${2:-$tarball_src_dir}")" # Make sure needed packages exist deps=( "$tarball_src_dir"/installinator.tar - "$tarball_src_dir"/maghemite.tar + "$tarball_src_dir"/mg-ddm-gz.tar ) for dep in "${deps[@]}"; do if [[ ! -e $dep ]]; then @@ -44,7 +44,7 @@ cd - pkg_dir="$tmp_trampoline/root/opt/oxide/mg-ddm" mkdir -p "$pkg_dir" cd "$pkg_dir" -tar -xvfz "$tarball_src_dir/maghemite.tar" +tar -xvfz "$tarball_src_dir/mg-ddm-gz.tar" cd - # Create the final output and we're done diff --git a/tools/ci_download_maghemite_mgd b/tools/ci_download_maghemite_mgd new file mode 100755 index 0000000000..eff680d7fd --- /dev/null +++ b/tools/ci_download_maghemite_mgd @@ -0,0 +1,168 @@ +#!/bin/bash + +# +# ci_download_maghemite_mgd: fetches the maghemite mgd binary tarball, unpacks +# it, and creates a copy called mgd, all in the current directory +# + +set -o pipefail +set -o xtrace +set -o errexit + +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +ARG0="$(basename "${BASH_SOURCE[0]}")" + +source "$SOURCE_DIR/maghemite_mgd_checksums" +source "$SOURCE_DIR/maghemite_mg_openapi_version" + +TARGET_DIR="out" +# Location where intermediate artifacts are downloaded / unpacked. +DOWNLOAD_DIR="$TARGET_DIR/downloads" +# Location where the final mgd directory should end up. +DEST_DIR="./$TARGET_DIR/mgd" +BIN_DIR="$DEST_DIR/root/opt/oxide/mgd/bin" + +ARTIFACT_URL="https://buildomat.eng.oxide.computer/public/file" + +REPO='oxidecomputer/maghemite' +PACKAGE_BASE_URL="$ARTIFACT_URL/$REPO/image/$COMMIT" + +function main +{ + # + # Process command-line arguments. We generally don't expect any, but + # we allow callers to specify a value to override OSTYPE, just for + # testing. + # + if [[ $# != 0 ]]; then + CIDL_OS="$1" + shift + else + CIDL_OS="$OSTYPE" + fi + + if [[ $# != 0 ]]; then + echo "unexpected arguments" >&2 + exit 2 + fi + + # Configure this program + configure_os "$CIDL_OS" + + CIDL_SHA256FUNC="do_sha256sum" + TARBALL_FILENAME="mgd.tar.gz" + PACKAGE_URL="$PACKAGE_BASE_URL/$TARBALL_FILENAME" + TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME" + + # Download the file. + echo "URL: $PACKAGE_URL" + echo "Local file: $TARBALL_FILE" + + mkdir -p "$DOWNLOAD_DIR" + mkdir -p "$DEST_DIR" + + fetch_and_verify + + do_untar "$TARBALL_FILE" + + do_assemble + + $SET_BINARIES +} + +function fail +{ + echo "$ARG0: $@" >&2 + exit 1 +} + +function configure_os +{ + echo "current directory: $PWD" + echo "configuring based on OS: \"$1\"" + case "$1" in + linux-gnu*) + SET_BINARIES="linux_binaries" + ;; + solaris*) + SET_BINARIES="" + ;; + *) + echo "WARNING: binaries for $1 are not published by maghemite" + echo "Dynamic routing apis will be unavailable" + SET_BINARIES="unsupported_os" + ;; + esac +} + +function do_download_curl +{ + curl --silent --show-error --fail --location --output "$2" "$1" +} + +function do_sha256sum +{ + sha256sum < "$1" | awk '{print $1}' +} + +function do_untar +{ + tar xzf "$1" -C "$DOWNLOAD_DIR" +} + +function do_assemble +{ + rm -r "$DEST_DIR" || true + mkdir "$DEST_DIR" + cp -r "$DOWNLOAD_DIR/root" "$DEST_DIR/root" +} + +function fetch_and_verify +{ + local DO_DOWNLOAD="true" + if [[ -f "$TARBALL_FILE" ]]; then + # If the file exists with a valid checksum, we can skip downloading. + calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \ + fail "failed to calculate sha256sum" + if [[ "$calculated_sha256" == "$CIDL_SHA256" ]]; then + DO_DOWNLOAD="false" + fi + fi + + if [ "$DO_DOWNLOAD" == "true" ]; then + echo "Downloading..." + do_download_curl "$PACKAGE_URL" "$TARBALL_FILE" || \ + fail "failed to download file" + + # Verify the sha256sum. + calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \ + fail "failed to calculate sha256sum" + if [[ "$calculated_sha256" != "$CIDL_SHA256" ]]; then + fail "sha256sum mismatch \ + (expected $CIDL_SHA256, found $calculated_sha256)" + fi + fi + +} + +function linux_binaries +{ + PACKAGE_BASE_URL="$ARTIFACT_URL/$REPO/linux/$COMMIT" + CIDL_SHA256="$MGD_LINUX_SHA256" + CIDL_SHA256FUNC="do_sha256sum" + TARBALL_FILENAME="mgd" + PACKAGE_URL="$PACKAGE_BASE_URL/$TARBALL_FILENAME" + TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME" + fetch_and_verify + chmod +x "$DOWNLOAD_DIR/mgd" + cp "$DOWNLOAD_DIR/mgd" "$BIN_DIR" +} + +function unsupported_os +{ + mkdir -p "$BIN_DIR" + echo "echo 'unsupported os' && exit 1" >> "$BIN_DIR/dpd" + chmod +x "$BIN_DIR/dpd" +} + +main "$@" diff --git a/tools/ci_download_maghemite_openapi b/tools/ci_download_maghemite_openapi index 37ff4f5547..db53f68d2c 100755 --- a/tools/ci_download_maghemite_openapi +++ b/tools/ci_download_maghemite_openapi @@ -15,10 +15,7 @@ TARGET_DIR="out" # Location where intermediate artifacts are downloaded / unpacked. DOWNLOAD_DIR="$TARGET_DIR/downloads" -source "$SOURCE_DIR/maghemite_openapi_version" -URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/openapi/$COMMIT/ddm-admin.json" -LOCAL_FILE="$DOWNLOAD_DIR/ddm-admin-$COMMIT.json" function main { @@ -83,4 +80,14 @@ function do_sha256sum $SHA < "$1" | awk '{print $1}' } +source "$SOURCE_DIR/maghemite_ddm_openapi_version" +URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/openapi/$COMMIT/ddm-admin.json" +LOCAL_FILE="$DOWNLOAD_DIR/ddm-admin-$COMMIT.json" + +main "$@" + +source "$SOURCE_DIR/maghemite_mg_openapi_version" +URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/openapi/$COMMIT/mg-admin.json" +LOCAL_FILE="$DOWNLOAD_DIR/mg-admin-$COMMIT.json" + main "$@" diff --git a/tools/ci_download_softnpu_machinery b/tools/ci_download_softnpu_machinery index 7975a310f0..cb5ea40210 100755 --- a/tools/ci_download_softnpu_machinery +++ b/tools/ci_download_softnpu_machinery @@ -15,7 +15,7 @@ OUT_DIR="out/npuzone" # Pinned commit for softnpu ASIC simulator SOFTNPU_REPO="softnpu" -SOFTNPU_COMMIT="eb27e6a00f1082c9faac7cf997e57d0609f7a309" +SOFTNPU_COMMIT="c1c42398c82b0220c8b5fa3bfba9c7a3bcaa0943" # This is the softnpu ASIC simulator echo "fetching npuzone" diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index 95c2aa63df..248cbcde73 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -44,6 +44,9 @@ function ensure_simulated_links { if [[ -z "$(get_vnic_name_if_exists "sc0_1")" ]]; then dladm create-vnic -t "sc0_1" -l "$PHYSICAL_LINK" -m a8:e1:de:01:70:1d + if [[ -v PROMISC_FILT_OFF ]]; then + dladm set-linkprop -p promisc-filtered=off sc0_1 + fi fi success "Vnic sc0_1 exists" } @@ -58,7 +61,8 @@ function ensure_softnpu_zone { out/npuzone/npuzone create sidecar \ --omicron-zone \ --ports sc0_0,tfportrear0_0 \ - --ports sc0_1,tfportqsfp0_0 + --ports sc0_1,tfportqsfp0_0 \ + --sidecar-lite-branch omicron-tracking } "$SOURCE_DIR"/scrimlet/softnpu-init.sh success "softnpu zone exists" diff --git a/tools/delete-reservoir.sh b/tools/delete-reservoir.sh new file mode 100755 index 0000000000..77e814f0c7 --- /dev/null +++ b/tools/delete-reservoir.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +size=`pfexec /usr/lib/rsrvrctl -q | grep Free | awk '{print $3}'` +let x=$size/1024 + +pfexec /usr/lib/rsrvrctl -r $x diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index b1f210a647..9a2ea85ac0 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="7712104585266a2898da38c1345210ad26f9e71d" +COMMIT="c0cbc39b55fac54b95468304c497e00f3d3cf686" SHA2="cb3f0cfbe6216d2441d34e0470252e0fb142332e47b33b65c24ef7368a694b6d" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index 9538bc0d00..fe52c59381 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="486b0b016c0df06947810b90f3a3dd40423f0ee6f255ed079dc8e5618c9a7281" -CIDL_SHA256_LINUX_DPD="af97aaf7e1046a5c651d316c384171df6387b4c54c8ae4a3ef498e532eaa5a4c" -CIDL_SHA256_LINUX_SWADM="909e400dcc9880720222c6dc3919404d83687f773f668160f66f38b51a81c188" +CIDL_SHA256_ILLUMOS="3706e0e8230b7f76407ec0acea9020b9efc7d6c78b74c304102fd8e62cac6760" +CIDL_SHA256_LINUX_DPD="b275a1c688eae1024b9ce1cbb766a66e37072e84b4a6cbc18746c903739ccf51" +CIDL_SHA256_LINUX_SWADM="7e604cc4b67c1a711a63ece2a8d0e2e7c8ef2b9ac6bb433b3c2e02f5f66018ba" diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index 62603ecac7..d3ecd8eaa8 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -197,6 +197,10 @@ retry ./tools/ci_download_dendrite_openapi # asic and running dendrite instance retry ./tools/ci_download_dendrite_stub +# Download mgd. This is required to run tests that invovle dynamic external +# routing +retry ./tools/ci_download_maghemite_mgd + # Download transceiver-control. This is used as the source for the # xcvradm binary which is bundled with the switch zone. retry ./tools/ci_download_transceiver_control diff --git a/tools/install_runner_prerequisites.sh b/tools/install_runner_prerequisites.sh index 7ece993bc9..42347f518d 100755 --- a/tools/install_runner_prerequisites.sh +++ b/tools/install_runner_prerequisites.sh @@ -105,6 +105,7 @@ function install_packages { 'pkg-config' 'brand/omicron1/tools' 'library/libxmlsec1' + 'chrony' ) # Install/update the set of packages. @@ -119,13 +120,15 @@ function install_packages { exit "$rc" fi + pfexec svcadm enable chrony + pkg list -v "${packages[@]}" elif [[ "${HOST_OS}" == "Linux" ]]; then packages=( 'ca-certificates' 'libpq5' 'libsqlite3-0' - 'libssl1.1' + 'libssl3' 'libxmlsec1-openssl' ) sudo apt-get update diff --git a/tools/maghemite_openapi_version b/tools/maghemite_ddm_openapi_version similarity index 59% rename from tools/maghemite_openapi_version rename to tools/maghemite_ddm_openapi_version index 8f84b30cb1..a315c31d4b 100644 --- a/tools/maghemite_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="12703675393459e74139f8140e0b3c4c4f129d5d" +COMMIT="2f25a2005521f643317879b46692141b4127608a" SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version new file mode 100644 index 0000000000..acd5a5f546 --- /dev/null +++ b/tools/maghemite_mg_openapi_version @@ -0,0 +1,2 @@ +COMMIT="2f25a2005521f643317879b46692141b4127608a" +SHA2="d0f7611e5ecd049b0f83bcfa843942401f155a0be36d9a2dfd73b8341d5f816e" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums new file mode 100644 index 0000000000..45fbec5274 --- /dev/null +++ b/tools/maghemite_mgd_checksums @@ -0,0 +1,2 @@ +CIDL_SHA256="16878501f5440590674acd82bee6ce5dcf3d1326531c25064dd9c060ab6440a4" +MGD_LINUX_SHA256="45e5ddc9d81cfcb94917f9c58942c3a7211fb34a3c563fbfc2434b0a97306b3d" diff --git a/tools/update_maghemite.sh b/tools/update_maghemite.sh index a4a9b1291e..eebece1aa5 100755 --- a/tools/update_maghemite.sh +++ b/tools/update_maghemite.sh @@ -15,8 +15,9 @@ function usage { } PACKAGES=( - "maghemite" + "mg-ddm-gz" "mg-ddm" + "mgd" ) REPO="oxidecomputer/maghemite" @@ -26,13 +27,14 @@ REPO="oxidecomputer/maghemite" function update_openapi { TARGET_COMMIT="$1" DRY_RUN="$2" - SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "ddm-admin.json" "openapi") + DAEMON="$3" + SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "${DAEMON}-admin.json" "openapi") OUTPUT=$(printf "COMMIT=\"%s\"\nSHA2=\"%s\"\n" "$TARGET_COMMIT" "$SHA") if [ -n "$DRY_RUN" ]; then OPENAPI_PATH="/dev/null" else - OPENAPI_PATH="$SOURCE_DIR/maghemite_openapi_version" + OPENAPI_PATH="$SOURCE_DIR/maghemite_${DAEMON}_openapi_version" fi echo "Updating Maghemite OpenAPI from: $TARGET_COMMIT" set -x @@ -40,6 +42,27 @@ function update_openapi { set +x } +function update_mgd { + TARGET_COMMIT="$1" + DRY_RUN="$2" + DAEMON="$3" + SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "image") + OUTPUT=$(printf "CIDL_SHA256=\"%s\"\n" "$SHA") + + SHA_LINUX=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "linux") + OUTPUT_LINUX=$(printf "MGD_LINUX_SHA256=\"%s\"\n" "$SHA_LINUX") + + if [ -n "$DRY_RUN" ]; then + MGD_PATH="/dev/null" + else + MGD_PATH="$SOURCE_DIR/maghemite_mgd_checksums" + fi + echo "Updating Maghemite mgd from: $TARGET_COMMIT" + set -x + echo "$OUTPUT\n$OUTPUT_LINUX" > $MGD_PATH + set +x +} + function main { TARGET_COMMIT="" DRY_RUN="" @@ -60,7 +83,9 @@ function main { TARGET_COMMIT=$(get_latest_commit_from_gh "$REPO" "$TARGET_COMMIT") install_toml2json do_update_packages "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${PACKAGES[@]}" - update_openapi "$TARGET_COMMIT" "$DRY_RUN" + update_openapi "$TARGET_COMMIT" "$DRY_RUN" ddm + update_openapi "$TARGET_COMMIT" "$DRY_RUN" mg + update_mgd "$TARGET_COMMIT" "$DRY_RUN" do_update_packages "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${PACKAGES[@]}" } diff --git a/update-engine/src/context.rs b/update-engine/src/context.rs index d232d931a2..cd85687cf9 100644 --- a/update-engine/src/context.rs +++ b/update-engine/src/context.rs @@ -223,7 +223,7 @@ impl StepContext { } } -/// Tracker for [`StepContext::add_nested_report`]. +/// Tracker for [`StepContext::send_nested_report`]. /// /// Nested event reports might contain events already seen in prior runs: /// `NestedEventBuffer` deduplicates those events such that only deltas are sent diff --git a/wicket/src/rack_setup/config_template.toml b/wicket/src/rack_setup/config_template.toml index 4b193a0c29..617b61fadc 100644 --- a/wicket/src/rack_setup/config_template.toml +++ b/wicket/src/rack_setup/config_template.toml @@ -40,18 +40,24 @@ bootstrap_sleds = [] # TODO: docs on network config [rack_network_config] +rack_subnet = "" infra_ip_first = "" infra_ip_last = "" -[[rack_network_config.uplinks]] +[[rack_network_config.ports]] +# Routes associated with this port. +# { nexthop = "1.2.3.4", destination = "0.0.0.0/0" } +routes = [] + +# Addresses associated with this port. +# "1.2.3.4/24" +addresses = [] + # Either `switch0` or `switch1`, matching the hardware. switch = "" -# IP address this uplink should use as its gateway. -gateway_ip = "" - # qsfp0, qsfp1, ... -uplink_port = "" +port = "" # `speed40_g`, `speed100_g`, ... uplink_port_speed = "" @@ -59,8 +65,14 @@ uplink_port_speed = "" # `none`, `firecode`, or `rs` uplink_port_fec = "" -# IP address and prefix for this uplink; e.g., `192.168.100.100/16` -uplink_cidr = "" +# A list of bgp peers +# { addr = "1.7.0.1", asn = 47, port = "qsfp0" } +bgp_peers = [] + +# Optional BGP configuration. Remove this section if not needed. +[[rack_network_config.bgp]] +# The autonomous system numer +asn = 0 -# VLAN ID for this uplink; omit if no VLAN ID is needed -uplink_vid = 1234 +# Prefixes to originate e.g., ["10.0.0.0/16"] +originate = [] diff --git a/wicket/src/rack_setup/config_toml.rs b/wicket/src/rack_setup/config_toml.rs index 5f0bb9e876..e087c9aa7c 100644 --- a/wicket/src/rack_setup/config_toml.rs +++ b/wicket/src/rack_setup/config_toml.rs @@ -18,7 +18,7 @@ use toml_edit::Value; use wicketd_client::types::BootstrapSledDescription; use wicketd_client::types::CurrentRssUserConfigInsensitive; use wicketd_client::types::IpRange; -use wicketd_client::types::RackNetworkConfig; +use wicketd_client::types::RackNetworkConfigV1; use wicketd_client::types::SpType; static TEMPLATE: &str = include_str!("config_template.toml"); @@ -176,7 +176,7 @@ fn build_sleds_array(sleds: &[BootstrapSledDescription]) -> Array { fn populate_network_table( table: &mut Table, - config: Option<&RackNetworkConfig>, + config: Option<&RackNetworkConfigV1>, ) { // Helper function to serialize enums into their appropriate string // representations. @@ -195,6 +195,7 @@ fn populate_network_table( }; for (property, value) in [ + ("rack_subnet", config.rack_subnet.to_string()), ("infra_ip_first", config.infra_ip_first.to_string()), ("infra_ip_last", config.infra_ip_last.to_string()), ] { @@ -202,20 +203,17 @@ fn populate_network_table( Value::String(Formatted::new(value)); } - // If `config.uplinks` is empty, we'll leave the template uplinks in place; - // otherwise, replace it with the user's uplinks. - if !config.uplinks.is_empty() { - *table.get_mut("uplinks").unwrap().as_array_of_tables_mut().unwrap() = + if !config.ports.is_empty() { + *table.get_mut("ports").unwrap().as_array_of_tables_mut().unwrap() = config - .uplinks + .ports .iter() .map(|cfg| { let mut uplink = Table::new(); - let mut last_key = None; + let mut _last_key = None; for (property, value) in [ ("switch", cfg.switch.to_string()), - ("gateway_ip", cfg.gateway_ip.to_string()), - ("uplink_port", cfg.uplink_port.to_string()), + ("port", cfg.port.to_string()), ( "uplink_port_speed", enum_to_toml_string(&cfg.uplink_port_speed), @@ -224,63 +222,121 @@ fn populate_network_table( "uplink_port_fec", enum_to_toml_string(&cfg.uplink_port_fec), ), - ("uplink_cidr", cfg.uplink_cidr.to_string()), ] { uplink.insert( property, Item::Value(Value::String(Formatted::new(value))), ); - last_key = Some(property); + _last_key = Some(property); } - if let Some(uplink_vid) = cfg.uplink_vid { - uplink.insert( - "uplink_vid", - Item::Value(Value::Integer(Formatted::new( - i64::from(uplink_vid), - ))), + let mut routes = Array::new(); + for r in &cfg.routes { + let mut route = InlineTable::new(); + route.insert( + "nexthop", + Value::String(Formatted::new( + r.nexthop.to_string(), + )), + ); + route.insert( + "destination", + Value::String(Formatted::new( + r.destination.to_string(), + )), ); - } else { - // Unwraps: We know `last_key` is `Some(_)`, because we - // set it in every iteration of the loop above, and we - // know it's present in `uplink` because we set it to - // the `property` we just inserted. - let last = uplink.get_mut(last_key.unwrap()).unwrap(); - - // Every item we insert is an `Item::Value`, so we can - // unwrap this conversion. - last.as_value_mut() - .unwrap() - .decor_mut() - .set_suffix("\n# uplink_vid ="); + routes.push(Value::InlineTable(route)); } + uplink.insert("routes", Item::Value(Value::Array(routes))); + let mut addresses = Array::new(); + for a in &cfg.addresses { + addresses + .push(Value::String(Formatted::new(a.to_string()))) + } + uplink.insert( + "addresses", + Item::Value(Value::Array(addresses)), + ); + + let mut peers = Array::new(); + for p in &cfg.bgp_peers { + let mut peer = InlineTable::new(); + peer.insert( + "addr", + Value::String(Formatted::new(p.addr.to_string())), + ); + peer.insert( + "asn", + Value::Integer(Formatted::new(p.asn as i64)), + ); + peer.insert( + "port", + Value::String(Formatted::new(p.port.to_string())), + ); + peers.push(Value::InlineTable(peer)); + } + uplink + .insert("bgp_peers", Item::Value(Value::Array(peers))); uplink }) .collect(); } + if !config.bgp.is_empty() { + *table.get_mut("bgp").unwrap().as_array_of_tables_mut().unwrap() = + config + .bgp + .iter() + .map(|cfg| { + let mut bgp = Table::new(); + bgp.insert( + "asn", + Item::Value(Value::Integer(Formatted::new( + cfg.asn as i64, + ))), + ); + + let mut originate = Array::new(); + for o in &cfg.originate { + originate + .push(Value::String(Formatted::new(o.to_string()))); + } + bgp.insert( + "originate", + Item::Value(Value::Array(originate)), + ); + bgp + }) + .collect(); + } } #[cfg(test)] mod tests { use super::*; - use omicron_common::api::internal::shared::RackNetworkConfig as InternalRackNetworkConfig; + use omicron_common::api::internal::shared::RackNetworkConfigV1 as InternalRackNetworkConfig; use std::net::Ipv6Addr; use wicket_common::rack_setup::PutRssUserConfigInsensitive; use wicketd_client::types::Baseboard; + use wicketd_client::types::BgpConfig; + use wicketd_client::types::BgpPeerConfig; + use wicketd_client::types::PortConfigV1; use wicketd_client::types::PortFec; use wicketd_client::types::PortSpeed; + use wicketd_client::types::RouteConfig; use wicketd_client::types::SpIdentifier; use wicketd_client::types::SwitchLocation; - use wicketd_client::types::UplinkConfig; fn put_config_from_current_config( value: CurrentRssUserConfigInsensitive, ) -> PutRssUserConfigInsensitive { + use omicron_common::api::internal::shared::BgpConfig as InternalBgpConfig; + use omicron_common::api::internal::shared::BgpPeerConfig as InternalBgpPeerConfig; + use omicron_common::api::internal::shared::PortConfigV1 as InternalPortConfig; use omicron_common::api::internal::shared::PortFec as InternalPortFec; use omicron_common::api::internal::shared::PortSpeed as InternalPortSpeed; + use omicron_common::api::internal::shared::RouteConfig as InternalRouteConfig; use omicron_common::api::internal::shared::SwitchLocation as InternalSwitchLocation; - use omicron_common::api::internal::shared::UplinkConfig as InternalUplinkConfig; let rnc = value.rack_network_config.unwrap(); @@ -310,14 +366,32 @@ mod tests { external_dns_ips: value.external_dns_ips, ntp_servers: value.ntp_servers, rack_network_config: InternalRackNetworkConfig { + rack_subnet: rnc.rack_subnet, infra_ip_first: rnc.infra_ip_first, infra_ip_last: rnc.infra_ip_last, - uplinks: rnc - .uplinks + ports: rnc + .ports .iter() - .map(|config| InternalUplinkConfig { - gateway_ip: config.gateway_ip, - uplink_port: config.uplink_port.clone(), + .map(|config| InternalPortConfig { + routes: config + .routes + .iter() + .map(|r| InternalRouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), + bgp_peers: config + .bgp_peers + .iter() + .map(|p| InternalBgpPeerConfig { + asn: p.asn, + port: p.port.clone(), + addr: p.addr, + }) + .collect(), + port: config.port.clone(), uplink_port_speed: match config.uplink_port_speed { PortSpeed::Speed0G => InternalPortSpeed::Speed0G, PortSpeed::Speed1G => InternalPortSpeed::Speed1G, @@ -340,8 +414,6 @@ mod tests { PortFec::None => InternalPortFec::None, PortFec::Rs => InternalPortFec::Rs, }, - uplink_cidr: config.uplink_cidr, - uplink_vid: config.uplink_vid, switch: match config.switch { SwitchLocation::Switch0 => { InternalSwitchLocation::Switch0 @@ -352,6 +424,14 @@ mod tests { }, }) .collect(), + bgp: rnc + .bgp + .iter() + .map(|config| InternalBgpConfig { + asn: config.asn, + originate: config.originate.clone(), + }) + .collect(), }, } } @@ -392,18 +472,30 @@ mod tests { )], external_dns_ips: vec!["10.0.0.1".parse().unwrap()], ntp_servers: vec!["ntp1.com".into(), "ntp2.com".into()], - rack_network_config: Some(RackNetworkConfig { + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: "fd00:1122:3344:01::/56".parse().unwrap(), infra_ip_first: "172.30.0.1".parse().unwrap(), infra_ip_last: "172.30.0.10".parse().unwrap(), - uplinks: vec![UplinkConfig { - gateway_ip: "172.30.0.10".parse().unwrap(), - uplink_cidr: "172.30.0.1/24".parse().unwrap(), + ports: vec![PortConfigV1 { + addresses: vec!["172.30.0.1/24".parse().unwrap()], + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: "172.30.0.10".parse().unwrap(), + }], + bgp_peers: vec![BgpPeerConfig { + asn: 47, + addr: "10.2.3.4".parse().unwrap(), + port: "port0".into(), + }], uplink_port_speed: PortSpeed::Speed400G, uplink_port_fec: PortFec::Firecode, - uplink_port: "port0".into(), - uplink_vid: None, + port: "port0".into(), switch: SwitchLocation::Switch0, }], + bgp: vec![BgpConfig { + asn: 47, + originate: vec!["10.0.0.0/16".parse().unwrap()], + }], }), }; let template = TomlTemplate::populate(&config).to_string(); diff --git a/wicket/src/ui/main.rs b/wicket/src/ui/main.rs index 42cc6bf587..58ea6c1771 100644 --- a/wicket/src/ui/main.rs +++ b/wicket/src/ui/main.rs @@ -23,7 +23,7 @@ use wicketd_client::types::GetLocationResponse; /// This structure allows us to maintain similar styling and navigation /// throughout wicket with a minimum of code. /// -/// Specific functionality is put inside [`Pane`]s, which can be customized +/// Specific functionality is put inside Panes, which can be customized /// as needed. pub struct MainScreen { #[allow(unused)] diff --git a/wicket/src/ui/panes/rack_setup.rs b/wicket/src/ui/panes/rack_setup.rs index 212ddff4da..086d01ce9d 100644 --- a/wicket/src/ui/panes/rack_setup.rs +++ b/wicket/src/ui/panes/rack_setup.rs @@ -695,56 +695,61 @@ fn rss_config_text<'a>( }; if let Some(cfg) = insensitive.rack_network_config.as_ref() { - for (i, uplink) in cfg.uplinks.iter().enumerate() { + for (i, uplink) in cfg.ports.iter().enumerate() { let mut items = vec![ vec![ - Span::styled(" • Switch : ", label_style), + Span::styled(" • Switch : ", label_style), Span::styled(uplink.switch.to_string(), ok_style), ], vec![ - Span::styled(" • Gateway IP : ", label_style), - Span::styled(uplink.gateway_ip.to_string(), ok_style), - ], - vec![ - Span::styled(" • Uplink CIDR : ", label_style), - Span::styled(uplink.uplink_cidr.to_string(), ok_style), - ], - vec![ - Span::styled(" • Uplink port : ", label_style), - Span::styled(uplink.uplink_port.clone(), ok_style), - ], - vec![ - Span::styled(" • Uplink port speed: ", label_style), + Span::styled(" • Speed : ", label_style), Span::styled( uplink.uplink_port_speed.to_string(), ok_style, ), ], vec![ - Span::styled(" • Uplink port FEC : ", label_style), + Span::styled(" • FEC : ", label_style), Span::styled(uplink.uplink_port_fec.to_string(), ok_style), ], ]; - if let Some(uplink_vid) = uplink.uplink_vid { - items.push(vec![ - Span::styled(" • Uplink VLAN id : ", label_style), - Span::styled(uplink_vid.to_string(), ok_style), - ]); - } else { - items.push(vec![ - Span::styled(" • Uplink VLAN id : ", label_style), - Span::styled("none", ok_style), - ]); - } + + let routes = uplink.routes.iter().map(|r| { + vec![ + Span::styled(" • Route : ", label_style), + Span::styled( + format!("{} -> {}", r.destination, r.nexthop), + ok_style, + ), + ] + }); + + let addresses = uplink.addresses.iter().map(|a| { + vec![ + Span::styled(" • Address : ", label_style), + Span::styled(a.to_string(), ok_style), + ] + }); + + let peers = uplink.bgp_peers.iter().map(|p| { + vec![ + Span::styled(" • BGP peer : ", label_style), + Span::styled(format!("{} ASN={}", p.addr, p.asn), ok_style), + ] + }); + + items.extend(routes); + items.extend(addresses); + items.extend(peers); append_list( &mut spans, - Cow::from(format!("Uplink {}: ", i + 1)), + Cow::from(format!("Port {}: ", i + 1)), items, ); } } else { - append_list(&mut spans, "Uplinks: ".into(), vec![]); + append_list(&mut spans, "Ports: ".into(), vec![]); } append_list( diff --git a/wicket/src/ui/wrap.rs b/wicket/src/ui/wrap.rs index 6cd5f7010a..9cd57d45d5 100644 --- a/wicket/src/ui/wrap.rs +++ b/wicket/src/ui/wrap.rs @@ -324,7 +324,7 @@ impl<'a> Fragment for StyledWord<'a> { /// Forcibly break spans wider than `line_width` into smaller spans. /// -/// This simply calls [`Span::break_apart`] on spans that are too wide. +/// This simply calls [`StyledWord::break_apart`] on spans that are too wide. fn break_words<'a, I>(spans: I, line_width: usize) -> Vec> where I: IntoIterator>, diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index f11fda9750..655f3bb803 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -25,6 +25,7 @@ hubtools.workspace = true http.workspace = true hyper.workspace = true illumos-utils.workspace = true +ipnetwork.workspace = true internal-dns.workspace = true itertools.workspace = true reqwest.workspace = true diff --git a/wicketd/src/installinator_progress.rs b/wicketd/src/installinator_progress.rs index ba3f743171..77baec2c94 100644 --- a/wicketd/src/installinator_progress.rs +++ b/wicketd/src/installinator_progress.rs @@ -165,7 +165,7 @@ enum RunningUpdate { /// Reports from the installinator have been received. /// /// This is an `UnboundedSender` to avoid cancel-safety issues (see - /// https://github.com/oxidecomputer/omicron/pull/3579). + /// ). ReportsReceived(watch::Sender), /// All messages have been received. diff --git a/wicketd/src/preflight_check/uplink.rs b/wicketd/src/preflight_check/uplink.rs index 58955d04d6..ebcba90645 100644 --- a/wicketd/src/preflight_check/uplink.rs +++ b/wicketd/src/preflight_check/uplink.rs @@ -17,12 +17,13 @@ use dpd_client::ClientState as DpdClientState; use either::Either; use illumos_utils::zone::SVCCFG; use illumos_utils::PFEXEC; +use ipnetwork::IpNetwork; use omicron_common::address::DENDRITE_PORT; +use omicron_common::api::internal::shared::PortConfigV1; use omicron_common::api::internal::shared::PortFec as OmicronPortFec; use omicron_common::api::internal::shared::PortSpeed as OmicronPortSpeed; use omicron_common::api::internal::shared::RackNetworkConfig; use omicron_common::api::internal::shared::SwitchLocation; -use omicron_common::api::internal::shared::UplinkConfig; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -32,7 +33,6 @@ use slog::Logger; use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; -use std::net::Ipv4Addr; use std::str::FromStr; use std::sync::Arc; use std::sync::Mutex; @@ -66,8 +66,6 @@ const CHRONYD: &str = "/usr/sbin/chronyd"; const IPADM: &str = "/usr/sbin/ipadm"; const ROUTE: &str = "/usr/sbin/route"; -const DPD_DEFAULT_IPV4_CIDR: &str = "0.0.0.0/0"; - pub(super) async fn run_local_uplink_preflight_check( network_config: RackNetworkConfig, dns_servers: Vec, @@ -90,7 +88,7 @@ pub(super) async fn run_local_uplink_preflight_check( let mut engine = UpdateEngine::new(log, sender); for uplink in network_config - .uplinks + .ports .iter() .filter(|uplink| uplink.switch == our_switch_location) { @@ -131,7 +129,7 @@ pub(super) async fn run_local_uplink_preflight_check( fn add_steps_for_single_local_uplink_preflight_check<'a>( engine: &mut UpdateEngine<'a>, dpd_client: &'a DpdClient, - uplink: &'a UplinkConfig, + uplink: &'a PortConfigV1, dns_servers: &'a [IpAddr], ntp_servers: &'a [String], dns_name_to_query: Option<&'a str>, @@ -153,7 +151,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( // Timeout we give to chronyd during the NTP check, in seconds. const CHRONYD_CHECK_TIMEOUT_SECS: &str = "30"; - let registrar = engine.for_component(uplink.uplink_port.clone()); + let registrar = engine.for_component(uplink.port.clone()); let prev_step = registrar .new_step( @@ -162,7 +160,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( |_cx| async { // Check that the port name is valid and that it has no links // configured already. - let port_id = PortId::from_str(&uplink.uplink_port) + let port_id = PortId::from_str(&uplink.port) .map_err(UplinkPreflightTerminalError::InvalidPortName)?; let links = dpd_client .link_list(&port_id) @@ -192,11 +190,11 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( { Ok(_response) => { let metadata = vec![format!( - "configured {}/{}: ip {}, gateway {}", + "configured {}/{}: ips {:#?}, routes {:#?}", *port_id, link_id.0, - uplink.uplink_cidr, - uplink.gateway_ip + uplink.addresses, + uplink.routes )]; StepSuccess::new((port_id, link_id)) .with_metadata(metadata) @@ -298,93 +296,99 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( // Tell the `uplink` service about the IP address we created on // the switch when configuring the uplink. let uplink_property = - UplinkProperty(format!("uplinks/{}_0", uplink.uplink_port)); - let uplink_cidr = uplink.uplink_cidr.to_string(); - - if let Err(err) = execute_command(&[ - SVCCFG, - "-s", - UPLINK_SMF_NAME, - "addpropvalue", - &uplink_property.0, - "astring:", - &uplink_cidr, - ]) - .await - { - return StepWarning::new( - Err(L2Failure::UplinkAddProperty(level1)), - format!("could not add uplink property: {err}"), - ) - .into(); - }; - - if let Err(err) = execute_command(&[ - SVCCFG, - "-s", - UPLINK_DEFAULT_SMF_NAME, - "refresh", - ]) - .await - { - return StepWarning::new( - Err(L2Failure::UplinkRefresh(level1, uplink_property)), - format!("could not add uplink property: {err}"), - ) - .into(); - }; - - // Wait for the `uplink` service to create the IP address. - let start_waiting_addr = Instant::now(); - 'waiting_for_addr: loop { - let ipadm_out = match execute_command(&[ - IPADM, - "show-addr", - "-p", - "-o", - "addr", + UplinkProperty(format!("uplinks/{}_0", uplink.port)); + + for addr in &uplink.addresses { + let uplink_cidr = addr.to_string(); + if let Err(err) = execute_command(&[ + SVCCFG, + "-s", + UPLINK_SMF_NAME, + "addpropvalue", + &uplink_property.0, + "astring:", + &uplink_cidr, ]) .await { - Ok(stdout) => stdout, - Err(err) => { - return StepWarning::new( - Err(L2Failure::RunIpadm( - level1, - uplink_property, - )), - format!("failed running ipadm: {err}"), - ) - .into(); - } + return StepWarning::new( + Err(L2Failure::UplinkAddProperty(level1)), + format!("could not add uplink property: {err}"), + ) + .into(); }; - for line in ipadm_out.split('\n') { - if line == uplink_cidr { - break 'waiting_for_addr; - } - } - - // We did not find `uplink_cidr` in the output of ipadm; - // sleep a bit and try again, unless we've been waiting too - // long already. - if start_waiting_addr.elapsed() < UPLINK_SVC_WAIT_TIMEOUT { - tokio::time::sleep(UPLINK_SVC_RETRY_DELAY).await; - } else { + if let Err(err) = execute_command(&[ + SVCCFG, + "-s", + UPLINK_DEFAULT_SMF_NAME, + "refresh", + ]) + .await + { return StepWarning::new( - Err(L2Failure::WaitingForHostAddr( + Err(L2Failure::UplinkRefresh( level1, uplink_property, )), - format!( - "timed out waiting for `uplink` to \ - create {uplink_cidr}" - ), + format!("could not add uplink property: {err}"), ) .into(); + }; + + // Wait for the `uplink` service to create the IP address. + let start_waiting_addr = Instant::now(); + 'waiting_for_addr: loop { + let ipadm_out = match execute_command(&[ + IPADM, + "show-addr", + "-p", + "-o", + "addr", + ]) + .await + { + Ok(stdout) => stdout, + Err(err) => { + return StepWarning::new( + Err(L2Failure::RunIpadm( + level1, + uplink_property, + )), + format!("failed running ipadm: {err}"), + ) + .into(); + } + }; + + for line in ipadm_out.split('\n') { + if line == uplink_cidr { + break 'waiting_for_addr; + } + } + + // We did not find `uplink_cidr` in the output of ipadm; + // sleep a bit and try again, unless we've been waiting too + // long already. + if start_waiting_addr.elapsed() + < UPLINK_SVC_WAIT_TIMEOUT + { + tokio::time::sleep(UPLINK_SVC_RETRY_DELAY).await; + } else { + return StepWarning::new( + Err(L2Failure::WaitingForHostAddr( + level1, + uplink_property, + )), + format!( + "timed out waiting for `uplink` to \ + create {uplink_cidr}" + ), + ) + .into(); + } } } - let metadata = vec![format!("configured {}", uplink_property.0)]; StepSuccess::new(Ok(L2Success { level1, uplink_property })) @@ -410,27 +414,29 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } }; - // Add the gateway as the default route in illumos. - if let Err(err) = execute_command(&[ - ROUTE, - "add", - "-inet", - "default", - &uplink.gateway_ip.to_string(), - ]) - .await - { - return StepWarning::new( - Err(RoutingFailure::HostDefaultRoute(level2)), - format!("could not add default route: {err}"), - ) - .into(); - }; + for r in &uplink.routes { + // Add the gateway as the default route in illumos. + if let Err(err) = execute_command(&[ + ROUTE, + "add", + "-inet", + &r.destination.to_string(), + &r.nexthop.to_string(), + ]) + .await + { + return StepWarning::new( + Err(RoutingFailure::HostDefaultRoute(level2)), + format!("could not add default route: {err}"), + ) + .into(); + }; + } StepSuccess::new(Ok(RoutingSuccess { level2 })) .with_metadata(vec![format!( - "added default route to {}", - uplink.gateway_ip + "added routes {:#?}", + uplink.routes, )]) .into() }, @@ -595,21 +601,24 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } }; - if remove_host_route { - execute_command(&[ - ROUTE, - "delete", - "-inet", - "default", - &uplink.gateway_ip.to_string(), - ]) - .await - .map_err(|err| { - UplinkPreflightTerminalError::RemoveHostRoute { - err, - gateway_ip: uplink.gateway_ip, - } - })?; + for r in &uplink.routes { + if remove_host_route { + execute_command(&[ + ROUTE, + "delete", + "-inet", + &r.destination.to_string(), + &r.nexthop.to_string(), + ]) + .await + .map_err(|err| { + UplinkPreflightTerminalError::RemoveHostRoute { + err, + destination: r.destination, + nexthop: r.nexthop, + } + })?; + } } StepSuccess::new(Ok(level2)).into() @@ -730,7 +739,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } fn build_port_settings( - uplink: &UplinkConfig, + uplink: &PortConfigV1, link_id: &LinkId, ) -> PortSettings { // Map from omicron_common types to dpd_client types @@ -758,10 +767,12 @@ fn build_port_settings( v6_routes: HashMap::new(), }; + let addrs = uplink.addresses.iter().map(|a| a.ip()).collect(); + port_settings.links.insert( link_id.to_string(), LinkSettings { - addrs: vec![IpAddr::V4(uplink.uplink_cidr.ip())], + addrs, params: LinkCreate { // TODO we should take these parameters too // https://github.com/oxidecomputer/omicron/issues/3061 @@ -773,14 +784,16 @@ fn build_port_settings( }, ); - port_settings.v4_routes.insert( - DPD_DEFAULT_IPV4_CIDR.parse().unwrap(), - RouteSettingsV4 { - link_id: link_id.0, - nexthop: uplink.gateway_ip, - vid: uplink.uplink_vid, - }, - ); + for r in &uplink.routes { + if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) = + (r.destination, r.nexthop) + { + port_settings.v4_routes.insert( + dst.to_string(), + RouteSettingsV4 { link_id: link_id.0, nexthop, vid: None }, + ); + } + } port_settings } @@ -890,8 +903,10 @@ pub(crate) enum UplinkPreflightTerminalError { err: DpdError, port_id: PortId, }, - #[error("failed to remove host OS route to gateway {gateway_ip}: {err}")] - RemoveHostRoute { err: String, gateway_ip: Ipv4Addr }, + #[error( + "failed to remove host OS route {destination} -> {nexthop}: {err}" + )] + RemoveHostRoute { err: String, destination: IpNetwork, nexthop: IpAddr }, #[error("failed to remove uplink SMF property {property:?}: {err}")] RemoveSmfProperty { property: String, err: String }, #[error("failed to refresh uplink service config: {0}")] diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index 1dc9f84985..a96acc56a0 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -453,18 +453,21 @@ impl From<&'_ CurrentRssConfig> for CurrentRssUserConfig { fn validate_rack_network_config( config: &RackNetworkConfig, -) -> Result { +) -> Result { + use bootstrap_agent_client::types::BgpConfig as BaBgpConfig; + use bootstrap_agent_client::types::BgpPeerConfig as BaBgpPeerConfig; + use bootstrap_agent_client::types::PortConfigV1 as BaPortConfigV1; use bootstrap_agent_client::types::PortFec as BaPortFec; use bootstrap_agent_client::types::PortSpeed as BaPortSpeed; + use bootstrap_agent_client::types::RouteConfig as BaRouteConfig; use bootstrap_agent_client::types::SwitchLocation as BaSwitchLocation; - use bootstrap_agent_client::types::UplinkConfig as BaUplinkConfig; use omicron_common::api::internal::shared::PortFec; use omicron_common::api::internal::shared::PortSpeed; use omicron_common::api::internal::shared::SwitchLocation; // Ensure that there is at least one uplink - if config.uplinks.is_empty() { - return Err(anyhow!("Must have at least one uplink configured")); + if config.ports.is_empty() { + return Err(anyhow!("Must have at least one port configured")); } // Make sure `infra_ip_first`..`infra_ip_last` is a well-defined range... @@ -475,34 +478,55 @@ fn validate_rack_network_config( }, )?; - // iterate through each UplinkConfig - for uplink_config in &config.uplinks { - // ... and check that it contains `uplink_ip`. - if uplink_config.uplink_cidr.ip() < infra_ip_range.first - || uplink_config.uplink_cidr.ip() > infra_ip_range.last - { - bail!( + // TODO this implies a single contiguous range for port IPs which is over + // constraining + // iterate through each port config + for port_config in &config.ports { + for addr in &port_config.addresses { + // ... and check that it contains `uplink_ip`. + if addr.ip() < infra_ip_range.first + || addr.ip() > infra_ip_range.last + { + bail!( "`uplink_cidr`'s IP address must be in the range defined by \ `infra_ip_first` and `infra_ip_last`" ); + } } } // TODO Add more client side checks on `rack_network_config` contents? - Ok(bootstrap_agent_client::types::RackNetworkConfig { + Ok(bootstrap_agent_client::types::RackNetworkConfigV1 { + rack_subnet: config.rack_subnet, infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, - uplinks: config - .uplinks + ports: config + .ports .iter() - .map(|config| BaUplinkConfig { - gateway_ip: config.gateway_ip, + .map(|config| BaPortConfigV1 { + port: config.port.clone(), + routes: config + .routes + .iter() + .map(|r| BaRouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), + bgp_peers: config + .bgp_peers + .iter() + .map(|p| BaBgpPeerConfig { + addr: p.addr, + asn: p.asn, + port: p.port.clone(), + }) + .collect(), switch: match config.switch { SwitchLocation::Switch0 => BaSwitchLocation::Switch0, SwitchLocation::Switch1 => BaSwitchLocation::Switch1, }, - uplink_cidr: config.uplink_cidr, - uplink_port: config.uplink_port.clone(), uplink_port_speed: match config.uplink_port_speed { PortSpeed::Speed0G => BaPortSpeed::Speed0G, PortSpeed::Speed1G => BaPortSpeed::Speed1G, @@ -519,7 +543,14 @@ fn validate_rack_network_config( PortFec::None => BaPortFec::None, PortFec::Rs => BaPortFec::Rs, }, - uplink_vid: config.uplink_vid, + }) + .collect(), + bgp: config + .bgp + .iter() + .map(|config| BaBgpConfig { + asn: config.asn, + originate: config.originate.clone(), }) .collect(), }) diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index b08f2612f1..3d9b195ae3 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -79,6 +79,7 @@ ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.18", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } +serde_json = { version = "1.0.107", features = ["raw_value"] } sha2 = { version = "0.10.7", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } @@ -171,6 +172,7 @@ ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.18", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } +serde_json = { version = "1.0.107", features = ["raw_value"] } sha2 = { version = "0.10.7", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } From e451ca59c52a819a9c00ab6ce1703cca6040d744 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Sat, 21 Oct 2023 00:21:43 -0700 Subject: [PATCH 70/89] bootstore: don't use hardcoded ports in tests (#4304) Fix some test flakiness by not hardcoding specific ports to use. --------- Co-authored-by: Andrew J. Stone --- bootstore/src/schemes/v0/peer.rs | 638 ++++++++++++++++++++----------- 1 file changed, 408 insertions(+), 230 deletions(-) diff --git a/bootstore/src/schemes/v0/peer.rs b/bootstore/src/schemes/v0/peer.rs index 7d29e2397a..3d273e60eb 100644 --- a/bootstore/src/schemes/v0/peer.rs +++ b/bootstore/src/schemes/v0/peer.rs @@ -91,6 +91,9 @@ pub enum NodeApiRequest { /// These are generated from DDM prefixes learned by the bootstrap agent. PeerAddresses(BTreeSet), + /// Get the local [`SocketAddrV6`] the node is listening on. + GetAddress { responder: oneshot::Sender }, + /// Get the status of this node GetStatus { responder: oneshot::Sender }, @@ -175,6 +178,17 @@ impl NodeHandle { Ok(()) } + /// Get the address of this node + pub async fn get_address(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.tx + .send(NodeApiRequest::GetAddress { responder: tx }) + .await + .map_err(|_| NodeRequestError::Send)?; + let res = rx.await?; + Ok(res) + } + /// Get the status of this node pub async fn get_status(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -361,6 +375,11 @@ impl Node { let mut interval = interval(self.config.time_per_tick); interval.set_missed_tick_behavior(MissedTickBehavior::Delay); let listener = TcpListener::bind(&self.config.addr).await.unwrap(); + // If the config didn't specify a port, let's update it + // with the actual port we binded to on our listener. + if self.config.addr.port() == 0 { + self.config.addr.set_port(listener.local_addr().unwrap().port()); + } while !self.shutdown { tokio::select! { res = listener.accept() => self.on_accept(res).await, @@ -487,6 +506,9 @@ impl Node { info!(self.log, "Updated Peer Addresses: {peers:?}"); self.manage_connections(peers).await; } + NodeApiRequest::GetAddress { responder } => { + let _ = responder.send(self.config.addr); + } NodeApiRequest::GetStatus { responder } => { let status = Status { fsm_ledger_generation: self.fsm_ledger_generation, @@ -1025,11 +1047,11 @@ mod tests { use super::*; use camino_tempfile::Utf8TempDir; use slog::Drain; - use tokio::time::sleep; + use tokio::{task::JoinHandle, time::sleep}; use uuid::Uuid; fn initial_members() -> BTreeSet { - [("a", "1"), ("b", "1"), ("c", "1")] + [("a", "0"), ("b", "1"), ("c", "2")] .iter() .map(|(id, model)| { Baseboard::new_pc(id.to_string(), model.to_string()) @@ -1037,56 +1059,10 @@ mod tests { .collect() } - fn initial_config(tempdir: &Utf8TempDir, port_start: u16) -> Vec { - initial_members() - .into_iter() - .enumerate() - .map(|(i, id)| { - let fsm_file = format!("test-{i}-fsm-state-ledger"); - let network_file = format!("test-{i}-network-config-ledger"); - Config { - id, - addr: format!("[::1]:{}{}", port_start, i).parse().unwrap(), - time_per_tick: Duration::from_millis(20), - learn_timeout: Duration::from_secs(5), - rack_init_timeout: Duration::from_secs(10), - rack_secret_request_timeout: Duration::from_secs(1), - fsm_state_ledger_paths: vec![tempdir - .path() - .join(&fsm_file)], - network_config_ledger_paths: vec![tempdir - .path() - .join(&network_file)], - } - }) - .collect() - } - fn learner_id(n: usize) -> Baseboard { Baseboard::new_pc("learner".to_string(), n.to_string()) } - fn learner_config( - tempdir: &Utf8TempDir, - n: usize, - port_start: u16, - ) -> Config { - let fsm_file = format!("test-learner-{n}-fsm-state-ledger"); - let network_file = format!("test-{n}-network-config-ledger"); - Config { - id: learner_id(n), - addr: format!("[::1]:{}{}", port_start, 3).parse().unwrap(), - time_per_tick: Duration::from_millis(20), - learn_timeout: Duration::from_secs(5), - rack_init_timeout: Duration::from_secs(10), - rack_secret_request_timeout: Duration::from_secs(1), - fsm_state_ledger_paths: vec![tempdir.path().join(&fsm_file)], - network_config_ledger_paths: vec![tempdir - .path() - .join(&network_file)], - } - } - fn log() -> slog::Logger { let decorator = slog_term::PlainDecorator::new(slog_term::TestStdoutWriter); @@ -1095,191 +1071,416 @@ mod tests { slog::Logger::root(drain, o!()) } - #[tokio::test] - async fn basic_3_nodes() { - let port_start = 3333; - let tempdir = Utf8TempDir::new().unwrap(); - let log = log(); - let config = initial_config(&tempdir, port_start); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let (mut node1, handle1) = Node::new(config[1].clone(), &log).await; - let (mut node2, handle2) = Node::new(config[2].clone(), &log).await; - - let jh0 = tokio::spawn(async move { - node0.run().await; - }); - let jh1 = tokio::spawn(async move { - node1.run().await; - }); - let jh2 = tokio::spawn(async move { - node2.run().await; - }); + struct TestNode { + log: Logger, + config: Config, + node_handles: Option<(NodeHandle, JoinHandle<()>)>, + } + + impl TestNode { + fn new(config: Config, log: Logger) -> TestNode { + TestNode { config, log, node_handles: None } + } + + async fn start_node(&mut self) { + // Node must have previously been shutdown (or never started) + assert!( + self.node_handles.is_none(), + "node ({}) already running", + self.config.id + ); + + // Reset port to pick any available + self.config.addr.set_port(0); + + // (Re-)create node with existing config and its persistent state (if any) + let (mut node, handle) = + Node::new(self.config.clone(), &self.log).await; + let jh = tokio::spawn(async move { + node.run().await; + }); + + // Grab assigned port + let port = handle + .get_address() + .await + .unwrap_or_else(|err| { + panic!( + "failed to get local address of node ({}): {err}", + self.config.id + ) + }) + .port(); + self.config.addr.set_port(port); + + self.node_handles = Some((handle, jh)); + } + + async fn shutdown_node(&mut self) { + let (handle, jh) = self.node_handles.take().unwrap_or_else(|| { + panic!("node ({}) not active", self.config.id) + }); + // Signal to the node it should shutdown + handle.shutdown().await.unwrap_or_else(|err| { + panic!("node ({}) failed to shutdown: {err}", self.config.id) + }); + // and wait for its task to spin down. + jh.await.unwrap_or_else(|err| { + panic!("node ({}) task failed: {err}", self.config.id) + }); + } + } + + struct TestNodes { + tempdir: Utf8TempDir, + log: Logger, + nodes: Vec, + learner: Option, + addrs: BTreeSet, + } + + impl TestNodes { + /// Create test nodes for the given set of members. + fn setup(initial_members: BTreeSet) -> TestNodes { + let tempdir = Utf8TempDir::new().unwrap(); + let log = log(); + let nodes = initial_members + .into_iter() + .enumerate() + .map(|(i, id)| { + let fsm_file = format!("test-{i}-fsm-state-ledger"); + let network_file = + format!("test-{i}-network-config-ledger"); + let config = Config { + id, + addr: SocketAddrV6::new( + std::net::Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ), + time_per_tick: Duration::from_millis(20), + learn_timeout: Duration::from_secs(5), + rack_init_timeout: Duration::from_secs(10), + rack_secret_request_timeout: Duration::from_secs(1), + fsm_state_ledger_paths: vec![tempdir + .path() + .join(&fsm_file)], + network_config_ledger_paths: vec![tempdir + .path() + .join(&network_file)], + }; + + TestNode::new(config, log.clone()) + }) + .collect(); + TestNodes { + tempdir, + log, + nodes, + learner: None, // No initial learner node + addrs: BTreeSet::new(), + } + } + + /// (Re-)start the given node and update peer addresses for everyone + async fn start_node(&mut self, i: usize) { + let node = &mut self.nodes[i]; + node.start_node().await; + self.addrs.insert(node.config.addr); + self.load_all_peer_addresses().await; + } + + // Stop the given node and update peer addresses for everyone + async fn shutdown_node(&mut self, i: usize) { + let node = &mut self.nodes[i]; + let addr = node.config.addr; + node.shutdown_node().await; + self.addrs.remove(&addr); + self.load_all_peer_addresses().await; + } + + /// Stop all active nodes (including the learner, if present). + async fn shutdown_all(&mut self) { + let nodes = self + .nodes + .iter_mut() + .chain(&mut self.learner) + .filter(|node| node.node_handles.is_some()); + for node in nodes { + node.shutdown_node().await; + } + self.addrs.clear(); + self.learner = None; + } + + /// Configure new learner node + async fn add_learner(&mut self, n: usize) { + assert!( + self.learner.is_none(), + "learner node already configured ({})", + self.learner.as_ref().unwrap().config.id + ); + + let fsm_file = format!("test-learner-{n}-fsm-state-ledger"); + let network_file = format!("test-{n}-network-config-ledger"); + let config = Config { + id: learner_id(n), + addr: SocketAddrV6::new(std::net::Ipv6Addr::LOCALHOST, 0, 0, 0), + time_per_tick: Duration::from_millis(20), + learn_timeout: Duration::from_secs(5), + rack_init_timeout: Duration::from_secs(10), + rack_secret_request_timeout: Duration::from_secs(1), + fsm_state_ledger_paths: vec![self + .tempdir + .path() + .join(&fsm_file)], + network_config_ledger_paths: vec![self + .tempdir + .path() + .join(&network_file)], + }; + + self.learner = Some(TestNode::new(config, self.log.clone())); + } - // Inform each node about the known addresses - let mut addrs: BTreeSet<_> = config.iter().map(|c| c.addr).collect(); - for handle in [&handle0, &handle1, &handle2] { - let _ = handle.load_peer_addresses(addrs.clone()).await; + /// Start a configured learner node and update peer addresses for everyone + async fn start_learner(&mut self) { + let learner = + self.learner.as_mut().expect("no learner node configured"); + learner.start_node().await; + let learner_addr = learner.config.addr; + + // Inform the learner and other nodes about all addresses including + // the learner. This simulates DDM discovery. + self.addrs.insert(learner_addr); + self.load_all_peer_addresses().await; } + /// Stop the learner node (but leave it configured) and update peer addresses for everyone + /// Can also optionally wipe the ledger persisted on disk. + async fn shutdown_learner(&mut self, wipe_ledger: bool) { + let learner = + self.learner.as_mut().expect("no learner node configured"); + let addr = learner.config.addr; + learner.shutdown_node().await; + + if wipe_ledger { + std::fs::remove_file(&learner.config.fsm_state_ledger_paths[0]) + .expect("failed to remove ledger"); + } + + // Update peer addresses + self.addrs.remove(&addr); + self.load_all_peer_addresses().await; + } + + /// Remove a configured learner node + async fn remove_learner(&mut self) { + // Shutdown the node if it's running + if matches!( + self.learner, + Some(TestNode { node_handles: Some(_), .. }) + ) { + self.shutdown_learner(false).await; + } + let _ = self.learner.take().expect("no learner node configured"); + } + + /// Inform each active node about its peers + async fn load_all_peer_addresses(&self) { + let nodes = + self.nodes.iter().chain(&self.learner).filter_map(|node| { + node.node_handles + .as_ref() + .map(|(h, _)| (&node.config.id, h)) + }); + for (id, node) in nodes { + node.load_peer_addresses(self.addrs.clone()).await.unwrap_or_else(|err| { + panic!("failed to update peer addresses for node ({id}): {err}") + }); + } + } + + /// Returns an iterator that yields the [`NodeHandle`]'s for all active + /// nodes (including the learner node, if present). + fn iter(&self) -> impl Iterator { + self.nodes + .iter() + .chain(&self.learner) + .filter_map(|node| node.node_handles.as_ref().map(|(h, _)| h)) + } + + /// To ensure deterministic learning of shares from node 0 which sorts first + /// we wait to ensure that the learner sees peer0 as connected before we + /// call `init_learner` + /// + /// Panics if the connection doesn't happen within `POLL_TIMEOUT` + async fn wait_for_learner_to_connect_to_node(&self, i: usize) { + const POLL_TIMEOUT: Duration = Duration::from_secs(5); + let start = Instant::now(); + loop { + let timeout = + POLL_TIMEOUT.saturating_sub(Instant::now() - start); + tokio::select! { + _ = sleep(timeout) => { + panic!("Learner not connected to node {i}"); + } + status = self[LEARNER].get_status() => { + let status = status.unwrap(); + let id = &self.nodes[i].config.id; + if status.connections.contains_key(id) { + break; + } + tokio::time::sleep(Duration::from_millis(1)).await; + } + } + } + } + } + + impl std::ops::Index for TestNodes { + type Output = NodeHandle; + + fn index(&self, index: usize) -> &Self::Output { + self.nodes[index] + .node_handles + .as_ref() + .map(|(handle, _)| handle) + .unwrap_or_else(|| panic!("node{index} not running")) + } + } + + // A little convenience to access the learner node in a similar + // manner as other nodes (indexing) but with a non-usize index. + const LEARNER: () = (); + impl std::ops::Index<()> for TestNodes { + type Output = NodeHandle; + + fn index(&self, _: ()) -> &Self::Output { + self.learner + .as_ref() + .expect("no learner node") + .node_handles + .as_ref() + .map(|(handle, _)| handle) + .expect("learner node not running") + } + } + + #[tokio::test] + async fn basic_3_nodes() { + // Create and start test nodes + let mut nodes = TestNodes::setup(initial_members()); + nodes.start_node(0).await; + nodes.start_node(1).await; + nodes.start_node(2).await; + let rack_uuid = RackUuid(Uuid::new_v4()); - handle0.init_rack(rack_uuid, initial_members()).await.unwrap(); + nodes[0].init_rack(rack_uuid, initial_members()).await.unwrap(); - let status = handle0.get_status().await; + let status = nodes[0].get_status().await; println!("Status = {status:?}"); // Ensure we can load the rack secret at all nodes - handle0.load_rack_secret().await.unwrap(); - handle1.load_rack_secret().await.unwrap(); - handle2.load_rack_secret().await.unwrap(); + for node in nodes.iter() { + node.load_rack_secret().await.unwrap(); + } // load the rack secret a second time on node0 - handle0.load_rack_secret().await.unwrap(); + nodes[0].load_rack_secret().await.unwrap(); // Shutdown the node2 and make sure we can still load the rack // secret (threshold=2) at node0 and node1 - handle2.shutdown().await.unwrap(); - jh2.await.unwrap(); - handle0.load_rack_secret().await.unwrap(); - handle1.load_rack_secret().await.unwrap(); - - // Add a learner node - let learner_conf = learner_config(&tempdir, 1, port_start); - let (mut learner, learner_handle) = - Node::new(learner_conf.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); - // Inform the learner and node0 and node1 about all addresses including - // the learner. This simulates DDM discovery - addrs.insert(learner_conf.addr); - let _ = learner_handle.load_peer_addresses(addrs.clone()).await; - let _ = handle0.load_peer_addresses(addrs.clone()).await; - let _ = handle1.load_peer_addresses(addrs.clone()).await; + nodes.shutdown_node(2).await; + nodes[0].load_rack_secret().await.unwrap(); + nodes[1].load_rack_secret().await.unwrap(); + + // Add and start a learner node + nodes.add_learner(1).await; + nodes.start_learner().await; // Tell the learner to go ahead and learn its share. - learner_handle.init_learner().await.unwrap(); + nodes[LEARNER].init_learner().await.unwrap(); // Shutdown node1 and show that we can still load the rack secret at // node0 and the learner, because threshold=2 and it never changes. - handle1.shutdown().await.unwrap(); - jh1.await.unwrap(); - handle0.load_rack_secret().await.unwrap(); - learner_handle.load_rack_secret().await.unwrap(); + nodes.shutdown_node(1).await; + nodes[0].load_rack_secret().await.unwrap(); + nodes[LEARNER].load_rack_secret().await.unwrap(); - // Now shutdown the learner and show that node0 cannot load the rack secret - learner_handle.shutdown().await.unwrap(); - learner_jh.await.unwrap(); - handle0.load_rack_secret().await.unwrap_err(); + // Now shutdown and remove the learner and show that node0 cannot load the rack secret + nodes.remove_learner().await; + nodes[0].load_rack_secret().await.unwrap_err(); - // Reload an node from persistent state and successfully reload the + // Reload a node from persistent state and successfully reload the // rack secret. - let (mut node1, handle1) = Node::new(config[1].clone(), &log).await; - let jh1 = tokio::spawn(async move { - node1.run().await; - }); - let _ = handle1.load_peer_addresses(addrs.clone()).await; - handle0.load_rack_secret().await.unwrap(); + nodes.start_node(1).await; + nodes[0].load_rack_secret().await.unwrap(); - // Add a second learner + // Grab the current generation numbers let peer0_gen = - handle0.get_status().await.unwrap().fsm_ledger_generation; + nodes[0].get_status().await.unwrap().fsm_ledger_generation; let peer1_gen = - handle1.get_status().await.unwrap().fsm_ledger_generation; - let learner_config = learner_config(&tempdir, 2, port_start); - let (mut learner, learner_handle) = - Node::new(learner_config.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); + nodes[1].get_status().await.unwrap().fsm_ledger_generation; + + // Add and start a second learner + nodes.add_learner(2).await; + nodes.start_learner().await; - // Inform the learner, node0, and node1 about all addresses including - // the learner. This simulates DDM discovery - addrs.insert(learner_config.addr); - let _ = learner_handle.load_peer_addresses(addrs.clone()).await; - let _ = handle0.load_peer_addresses(addrs.clone()).await; - let _ = handle1.load_peer_addresses(addrs.clone()).await; + // Wait for the learner to connect to node 0 + nodes.wait_for_learner_to_connect_to_node(0).await; // Tell the learner to go ahead and learn its share. - learner_handle.init_learner().await.unwrap(); + nodes[LEARNER].init_learner().await.unwrap(); // Get the new generation numbers let peer0_gen_new = - handle0.get_status().await.unwrap().fsm_ledger_generation; + nodes[0].get_status().await.unwrap().fsm_ledger_generation; let peer1_gen_new = - handle1.get_status().await.unwrap().fsm_ledger_generation; + nodes[1].get_status().await.unwrap().fsm_ledger_generation; - // Ensure only one of the peers generation numbers gets bumped - assert!( - (peer0_gen_new == peer0_gen && peer1_gen_new == peer1_gen + 1) - || (peer0_gen_new == peer0_gen + 1 - && peer1_gen_new == peer1_gen) - ); + // Ensure only peer 0's generation number gets bumped + assert_eq!(peer0_gen_new, peer0_gen + 1); + assert_eq!(peer1_gen_new, peer1_gen); + + // Now we can stop the learner, wipe its ledger, and restart it. + nodes.shutdown_learner(true).await; + nodes.start_learner().await; // Wipe the learner ledger, restart the learner and instruct it to // relearn its share, and ensure that the neither generation number gets - // bumped because persistence doesn't occur. - learner_handle.shutdown().await.unwrap(); - learner_jh.await.unwrap(); - std::fs::remove_file(&learner_config.fsm_state_ledger_paths[0]) - .unwrap(); - let (mut learner, learner_handle) = - Node::new(learner_config.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); - let _ = learner_handle.load_peer_addresses(addrs.clone()).await; - learner_handle.init_learner().await.unwrap(); + // bumped because persistence doesn't occur. But for that to happen + // we need to make sure the learner asks the same peer, which is node 0 since + // it sorts first based on its id which is of type `Baseboard`. + nodes.wait_for_learner_to_connect_to_node(0).await; + nodes[LEARNER].init_learner().await.unwrap(); + + // Ensure the peers' generation numbers didn't get bumped. The learner + // should've asked the same sled for a share first, which it already + // handed out. let peer0_gen_new_2 = - handle0.get_status().await.unwrap().fsm_ledger_generation; + nodes[0].get_status().await.unwrap().fsm_ledger_generation; let peer1_gen_new_2 = - handle1.get_status().await.unwrap().fsm_ledger_generation; - - // Ensure the peer's generation numbers don't get bumped. The learner - // will ask the same sled for a share first, which it already handed - // out. - assert!( - peer0_gen_new == peer0_gen_new_2 - && peer1_gen_new == peer1_gen_new_2 - ); + nodes[1].get_status().await.unwrap().fsm_ledger_generation; + assert_eq!(peer0_gen_new, peer0_gen_new_2); + assert_eq!(peer1_gen_new, peer1_gen_new_2); - // Shutdown the new learner, node0, and node1 - learner_handle.shutdown().await.unwrap(); - learner_jh.await.unwrap(); - handle0.shutdown().await.unwrap(); - jh0.await.unwrap(); - handle1.shutdown().await.unwrap(); - jh1.await.unwrap(); + // Shut it all down + nodes.shutdown_all().await; } #[tokio::test] async fn network_config() { - let port_start = 4444; - let tempdir = Utf8TempDir::new().unwrap(); - let log = log(); - let config = initial_config(&tempdir, port_start); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let (mut node1, handle1) = Node::new(config[1].clone(), &log).await; - let (mut node2, handle2) = Node::new(config[2].clone(), &log).await; - - let jh0 = tokio::spawn(async move { - node0.run().await; - }); - let jh1 = tokio::spawn(async move { - node1.run().await; - }); - let jh2 = tokio::spawn(async move { - node2.run().await; - }); - - // Inform each node about the known addresses - let mut addrs: BTreeSet<_> = config.iter().map(|c| c.addr).collect(); - for handle in [&handle0, &handle1, &handle2] { - let _ = handle.load_peer_addresses(addrs.clone()).await; - } + // Create and start test nodes + let mut nodes = TestNodes::setup(initial_members()); + nodes.start_node(0).await; + nodes.start_node(1).await; + nodes.start_node(2).await; // Ensure there is no network config at any of the nodes - for handle in [&handle0, &handle1, &handle2] { - assert_eq!(None, handle.get_network_config().await.unwrap()); + for node in nodes.iter() { + assert_eq!(None, node.get_network_config().await.unwrap()); } // Update the network config at node0 and ensure it has taken effect @@ -1287,10 +1488,10 @@ mod tests { generation: 1, blob: b"Some network data".to_vec(), }; - handle0.update_network_config(network_config.clone()).await.unwrap(); + nodes[0].update_network_config(network_config.clone()).await.unwrap(); assert_eq!( Some(&network_config), - handle0.get_network_config().await.unwrap().as_ref() + nodes[0].get_network_config().await.unwrap().as_ref() ); // Poll node1 and node2 until the network config update shows up @@ -1305,13 +1506,13 @@ mod tests { _ = sleep(timeout) => { panic!("Network config not replicated"); } - res = handle1.get_network_config(), if !node1_done => { + res = nodes[1].get_network_config(), if !node1_done => { if res.unwrap().as_ref() == Some(&network_config) { node1_done = true; continue; } } - res = handle2.get_network_config(), if !node2_done => { + res = nodes[2].get_network_config(), if !node2_done => { if res.unwrap().as_ref() == Some(&network_config) { node2_done = true; continue; @@ -1321,18 +1522,8 @@ mod tests { } // Bring a learner online - let learner_conf = learner_config(&tempdir, 1, port_start); - let (mut learner, learner_handle) = - Node::new(learner_conf.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); - // Inform the learner and other nodes about all addresses including - // the learner. This simulates DDM discovery. - addrs.insert(learner_conf.addr); - for handle in [&learner_handle, &handle0, &handle1, &handle2] { - let _ = handle.load_peer_addresses(addrs.clone()).await; - } + nodes.add_learner(1).await; + nodes.start_learner().await; // Poll the learner to ensure it gets the network config // Note that the learner doesn't even need to learn its share @@ -1345,7 +1536,7 @@ mod tests { _ = sleep(timeout) => { panic!("Network config not replicated"); } - res = learner_handle.get_network_config() => { + res = nodes[LEARNER].get_network_config() => { if res.unwrap().as_ref() == Some(&network_config) { done = true; } @@ -1355,34 +1546,26 @@ mod tests { // Stop node0, bring it back online and ensure it still sees the config // at generation 1 - handle0.shutdown().await.unwrap(); - jh0.await.unwrap(); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let jh0 = tokio::spawn(async move { - node0.run().await; - }); + nodes.shutdown_node(0).await; + nodes.start_node(0).await; assert_eq!( Some(&network_config), - handle0.get_network_config().await.unwrap().as_ref() + nodes[0].get_network_config().await.unwrap().as_ref() ); // Stop node0 again, update network config via node1, bring node0 back online, // and ensure all nodes see the latest configuration. + nodes.shutdown_node(0).await; let new_config = NetworkConfig { generation: 2, blob: b"Some more network data".to_vec(), }; - handle0.shutdown().await.unwrap(); - jh0.await.unwrap(); - handle1.update_network_config(new_config.clone()).await.unwrap(); + nodes[1].update_network_config(new_config.clone()).await.unwrap(); assert_eq!( Some(&new_config), - handle1.get_network_config().await.unwrap().as_ref() + nodes[1].get_network_config().await.unwrap().as_ref() ); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let jh0 = tokio::spawn(async move { - node0.run().await; - }); + nodes.start_node(0).await; let start = Instant::now(); // These should all resolve instantly, so no real need for a select, // which is getting tedious. @@ -1392,8 +1575,8 @@ mod tests { if Instant::now() - start > POLL_TIMEOUT { panic!("network config not replicated"); } - for h in [&handle0, &handle1, &handle2, &learner_handle] { - if h.get_network_config().await.unwrap().as_ref() + for node in nodes.iter() { + if node.get_network_config().await.unwrap().as_ref() != Some(&new_config) { // We need to try again @@ -1410,16 +1593,11 @@ mod tests { current_generation: 2, }); assert_eq!( - handle0.update_network_config(network_config).await, + nodes[0].update_network_config(network_config).await, expected ); // Shut it all down - for h in [handle0, handle1, handle2, learner_handle] { - let _ = h.shutdown().await; - } - for jh in [jh0, jh1, jh2, learner_jh] { - jh.await.unwrap(); - } + nodes.shutdown_all().await; } } From 3365c2b636a2e8c955f8dd50ed43def68f13e9f9 Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Sat, 21 Oct 2023 11:54:30 -0400 Subject: [PATCH 71/89] Update crucible and propolis (#4306) Bump crucible rev to pick up: - Add session_id to crucible logs - Don't check for a snapshot that's gone - Update Rust crate libc to 0.2.149 - Update Rust crate csv to 1.3.0 - Update Rust crate tokio to 1.33 - Add timeout and failure data collection of replay job Bump propolis rev to pick up: - Bump crucible rev to latest Also ran `cargo update -p libc -p tokio` --- Cargo.lock | 40 +++++++++++++++++++-------------------- Cargo.toml | 16 ++++++++-------- package-manifest.toml | 12 ++++++------ workspace-hack/Cargo.toml | 4 ++-- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06d5f2fb70..d99358a9ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -495,7 +495,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "bhyve_api_sys", "libc", @@ -505,7 +505,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "libc", "strum", @@ -1235,7 +1235,7 @@ dependencies = [ [[package]] name = "cpuid_profile_config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "propolis", "serde", @@ -1443,7 +1443,7 @@ dependencies = [ [[package]] name = "crucible" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "aes-gcm-siv", "anyhow", @@ -1488,7 +1488,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "anyhow", "chrono", @@ -1504,7 +1504,7 @@ dependencies = [ [[package]] name = "crucible-client-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "base64 0.21.4", "crucible-workspace-hack", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "crucible-common" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "anyhow", "atty", @@ -1545,7 +1545,7 @@ dependencies = [ [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "anyhow", "chrono", @@ -1562,7 +1562,7 @@ dependencies = [ [[package]] name = "crucible-protocol" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "anyhow", "bincode", @@ -1579,7 +1579,7 @@ dependencies = [ [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "crucible-workspace-hack", "libc", @@ -2029,7 +2029,7 @@ checksum = "7e1a8646b2c125eeb9a84ef0faa6d2d102ea0d5da60b824ade2743263117b848" [[package]] name = "dladm" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "libc", "strum", @@ -6642,7 +6642,7 @@ dependencies = [ [[package]] name = "propolis" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "anyhow", "bhyve_api", @@ -6675,7 +6675,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "async-trait", "base64 0.21.4", @@ -6699,7 +6699,7 @@ dependencies = [ [[package]] name = "propolis-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "anyhow", "async-trait", @@ -6751,7 +6751,7 @@ dependencies = [ [[package]] name = "propolis-server-config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "cpuid_profile_config", "serde", @@ -6763,7 +6763,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "schemars", "serde", @@ -9007,9 +9007,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -9803,7 +9803,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "viona_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "libc", "viona_api_sys", @@ -9812,7 +9812,7 @@ dependencies = [ [[package]] name = "viona_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "libc", ] diff --git a/Cargo.toml b/Cargo.toml index e9eea3c4ee..58d57d15b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,10 +165,10 @@ cookie = "0.16" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } -crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } +crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } curve25519-dalek = "4" datatest-stable = "0.1.3" display-error-chain = "0.1.1" @@ -284,9 +284,9 @@ pretty-hex = "0.3.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "334df299a56cd0d33e1227ed4ce4d2fe7478d541" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "334df299a56cd0d33e1227ed4ce4d2fe7478d541", features = [ "generated-migration" ] } -propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "334df299a56cd0d33e1227ed4ce4d2fe7478d541", default-features = false, features = ["mock-only"] } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2", features = [ "generated-migration" ] } +propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2", default-features = false, features = ["mock-only"] } proptest = "1.3.1" quote = "1.0" rand = "0.8.5" @@ -354,7 +354,7 @@ textwrap = "0.16.0" test-strategy = "0.2.1" thiserror = "1.0" tofino = { git = "http://github.com/oxidecomputer/tofino", branch = "main" } -tokio = "1.29" +tokio = "1.33.0" tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] } tokio-stream = "0.1.14" tokio-tungstenite = "0.18" diff --git a/package-manifest.toml b/package-manifest.toml index 3404f5f44f..5fc8114116 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -381,10 +381,10 @@ only_for_targets.image = "standard" # 3. Use source.type = "manual" instead of "prebuilt" source.type = "prebuilt" source.repo = "crucible" -source.commit = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source.commit = "da534e73380f3cc53ca0de073e1ea862ae32109b" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible.sha256.txt -source.sha256 = "010281ff5c3a0807c9e770d79264c954816a055aa482988d81e85ed98242e454" +source.sha256 = "572ac3b19e51b4e476266a62c2b7e06eff81c386cb48247c4b9f9b1e2ee81895" output.type = "zone" [package.crucible-pantry] @@ -392,10 +392,10 @@ service_name = "crucible_pantry" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source.commit = "da534e73380f3cc53ca0de073e1ea862ae32109b" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-pantry.sha256.txt -source.sha256 = "809936edff2957e761e49667d5477e34b7a862050b4e082a59fdc95096d3bdd5" +source.sha256 = "812269958e18f54d72bc10bb4fb81f26c084cf762da7fd98e63d58c689be9ad1" output.type = "zone" # Refer to @@ -406,10 +406,10 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source.commit = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "531e0654de94b6e805836c35aa88b8a1ac691184000a03976e2b7825061e904e" +source.sha256 = "aa1d9dc5c9117c100f9636901e8eec6679d7dfbf869c46b7f2873585f94a1b89" output.type = "zone" [package.mg-ddm-gz] diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 3d9b195ae3..45c6419d2a 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -91,7 +91,7 @@ syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extr syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } textwrap = { version = "0.16.0" } time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } -tokio = { version = "1.32.0", features = ["full", "test-util"] } +tokio = { version = "1.33.0", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } @@ -185,7 +185,7 @@ syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra textwrap = { version = "0.16.0" } time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } time-macros = { version = "0.2.13", default-features = false, features = ["formatting", "parsing"] } -tokio = { version = "1.32.0", features = ["full", "test-util"] } +tokio = { version = "1.33.0", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } From 132efaccb1534bdafb92318f6927002ecdfa68dd Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Sat, 21 Oct 2023 09:28:25 -0700 Subject: [PATCH 72/89] [nexus] Improve logging for schema changes (#4309) Improves visibility of logging, indicating: - What version are we upgrading into, throughout the process - Logging the names of SQL files used for upgrade, as they occur --- .../src/db/datastore/db_metadata.rs | 29 +++++++++++++------ nexus/db-queries/src/db/datastore/mod.rs | 3 +- nexus/tests/integration_tests/schema.rs | 15 +++++++--- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs index 9e4e8b1a48..0ae61a7c38 100644 --- a/nexus/db-queries/src/db/datastore/db_metadata.rs +++ b/nexus/db-queries/src/db/datastore/db_metadata.rs @@ -25,6 +25,17 @@ use std::str::FromStr; pub const EARLIEST_SUPPORTED_VERSION: &'static str = "1.0.0"; +/// Describes a single file containing a schema change, as SQL. +pub struct SchemaUpgradeStep { + pub path: Utf8PathBuf, + pub sql: String, +} + +/// Describes a sequence of files containing schema changes. +pub struct SchemaUpgrade { + pub steps: Vec, +} + /// Reads a "version directory" and reads all SQL changes into /// a result Vec. /// @@ -34,7 +45,7 @@ pub const EARLIEST_SUPPORTED_VERSION: &'static str = "1.0.0"; /// These are sorted lexicographically. pub async fn all_sql_for_version_migration>( path: P, -) -> Result, String> { +) -> Result { let target_dir = path.as_ref(); let mut up_sqls = vec![]; let entries = target_dir @@ -54,13 +65,12 @@ pub async fn all_sql_for_version_migration>( } up_sqls.sort(); - let mut result = vec![]; + let mut result = SchemaUpgrade { steps: vec![] }; for path in up_sqls.into_iter() { - result.push( - tokio::fs::read_to_string(&path) - .await - .map_err(|e| format!("Cannot read {path}: {e}"))?, - ); + let sql = tokio::fs::read_to_string(&path) + .await + .map_err(|e| format!("Cannot read {path}: {e}"))?; + result.steps.push(SchemaUpgradeStep { path: path.to_owned(), sql }); } Ok(result) } @@ -187,7 +197,8 @@ impl DataStore { ) .map_err(|e| format!("Invalid schema path: {}", e.display()))?; - let up_sqls = all_sql_for_version_migration(&target_dir).await?; + let schema_change = + all_sql_for_version_migration(&target_dir).await?; // Confirm the current version, set the "target_version" // column to indicate that a schema update is in-progress. @@ -205,7 +216,7 @@ impl DataStore { "target_version" => target_version.to_string(), ); - for sql in &up_sqls { + for SchemaUpgradeStep { path: _, sql } in &schema_change.steps { // Perform the schema change. self.apply_schema_update( ¤t_version, diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index f5283e263e..2dc1e69a6f 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -91,7 +91,8 @@ mod zpool; pub use address_lot::AddressLotCreateResult; pub use db_metadata::{ - all_sql_for_version_migration, EARLIEST_SUPPORTED_VERSION, + all_sql_for_version_migration, SchemaUpgrade, SchemaUpgradeStep, + EARLIEST_SUPPORTED_VERSION, }; pub use dns::DnsVersionUpdateBuilder; pub use instance::InstanceAndActiveVmm; diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index e75211b834..d79dd09fc1 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -89,6 +89,7 @@ async fn apply_update_as_transaction( match apply_update_as_transaction_inner(client, sql).await { Ok(()) => break, Err(err) => { + warn!(log, "Failed to apply update as transaction"; "err" => err.to_string()); client .batch_execute("ROLLBACK;") .await @@ -111,7 +112,9 @@ async fn apply_update( version: &str, times_to_apply: usize, ) { - info!(log, "Performing upgrade to {version}"); + let log = log.new(o!("target version" => version.to_string())); + info!(log, "Performing upgrade"); + let client = crdb.connect().await.expect("failed to connect"); // We skip this for the earliest supported version because these tables @@ -126,11 +129,15 @@ async fn apply_update( } let target_dir = Utf8PathBuf::from(SCHEMA_DIR).join(version); - let sqls = all_sql_for_version_migration(&target_dir).await.unwrap(); + let schema_change = + all_sql_for_version_migration(&target_dir).await.unwrap(); for _ in 0..times_to_apply { - for sql in sqls.iter() { - apply_update_as_transaction(log, &client, sql).await; + for nexus_db_queries::db::datastore::SchemaUpgradeStep { path, sql } in + &schema_change.steps + { + info!(log, "Applying sql schema upgrade step"; "path" => path.to_string()); + apply_update_as_transaction(&log, &client, sql).await; } } From aef679c4ae8f0921e34c59e480e1400a2775cc7e Mon Sep 17 00:00:00 2001 From: bnaecker Date: Sat, 21 Oct 2023 10:28:17 -0700 Subject: [PATCH 73/89] Limit size of the search range for VNIs when creating VPCs (#4298) - Fixes #4283. - Adds a relatively small limit to the `NextItem` query used for finding a free VNI during VPC creation. This limits the memory consumption to something very reasonable, but is big enough that we should be extremely unlikely to find _no_ available VNIs in the range. - Add an application-level retry loop when inserting _customer_ VPCs, which catches the unlikely event that there really are no VNIs available, and retries a few times. - Adds tests for the computation of the limited search range. - Adds tests for the actual exhaustion-detection and retry behavior. --- Cargo.lock | 1 + nexus/db-queries/Cargo.toml | 1 + nexus/db-queries/src/db/datastore/vpc.rs | 376 ++++++++++++++++++++--- nexus/db-queries/src/db/queries/vpc.rs | 215 ++++++++++++- nexus/db-queries/src/lib.rs | 19 ++ 5 files changed, 565 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d99358a9ad..7dbaa3e008 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4464,6 +4464,7 @@ dependencies = [ "serde_with", "sled-agent-client", "slog", + "static_assertions", "steno", "strum", "subprocess", diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index eaf3dc1295..c16c0f5319 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -47,6 +47,7 @@ serde_urlencoded.workspace = true serde_with.workspace = true sled-agent-client.workspace = true slog.workspace = true +static_assertions.workspace = true steno.workspace = true thiserror.workspace = true tokio = { workspace = true, features = [ "full" ] } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 14886ba018..6db99465a3 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -34,12 +34,15 @@ use crate::db::model::VpcUpdate; use crate::db::model::{Ipv4Net, Ipv6Net}; use crate::db::pagination::paginated; use crate::db::queries::vpc::InsertVpcQuery; +use crate::db::queries::vpc::VniSearchIter; use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery; use crate::db::queries::vpc_subnet::SubnetError; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; +use diesel::result::DatabaseErrorKind; +use diesel::result::Error as DieselError; use ipnetwork::IpNetwork; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; @@ -85,18 +88,23 @@ impl DataStore { SERVICES_VPC.clone(), Some(Vni(ExternalVni::SERVICES_VNI)), ); - let authz_vpc = self + let authz_vpc = match self .project_create_vpc_raw(opctx, &authz_project, vpc_query) .await - .map(|(authz_vpc, _)| authz_vpc) - .or_else(|e| match e { - Error::ObjectAlreadyExists { .. } => Ok(authz::Vpc::new( - authz_project.clone(), - *SERVICES_VPC_ID, - LookupType::ByName(SERVICES_VPC.identity.name.to_string()), - )), - _ => Err(e), - })?; + { + Ok(None) => { + let msg = "VNI exhaustion detected when creating built-in VPCs"; + error!(opctx.log, "{}", msg); + Err(Error::internal_error(msg)) + } + Ok(Some((authz_vpc, _))) => Ok(authz_vpc), + Err(Error::ObjectAlreadyExists { .. }) => Ok(authz::Vpc::new( + authz_project.clone(), + *SERVICES_VPC_ID, + LookupType::ByName(SERVICES_VPC.identity.name.to_string()), + )), + Err(e) => Err(e), + }?; // Also add the system router and internet gateway route @@ -287,22 +295,65 @@ impl DataStore { &self, opctx: &OpContext, authz_project: &authz::Project, - vpc: IncompleteVpc, + mut vpc: IncompleteVpc, ) -> Result<(authz::Vpc, Vpc), Error> { - self.project_create_vpc_raw( - opctx, - authz_project, - InsertVpcQuery::new(vpc), - ) - .await + // Generate an iterator that allows us to search the entire space of + // VNIs for this VPC, in manageable chunks to limit memory usage. + let vnis = VniSearchIter::new(vpc.vni.0); + for (i, vni) in vnis.enumerate() { + vpc.vni = Vni(vni); + let id = usdt::UniqueId::new(); + crate::probes::vni__search__range__start!(|| { + (&id, u32::from(vni), VniSearchIter::STEP_SIZE) + }); + match self + .project_create_vpc_raw( + opctx, + authz_project, + InsertVpcQuery::new(vpc.clone()), + ) + .await + { + Ok(Some((authz_vpc, vpc))) => { + crate::probes::vni__search__range__found!(|| { + (&id, u32::from(vpc.vni.0)) + }); + return Ok((authz_vpc, vpc)); + } + Err(e) => return Err(e), + Ok(None) => { + crate::probes::vni__search__range__empty!(|| (&id)); + debug!( + opctx.log, + "No VNIs available within current search range, retrying"; + "attempt" => i, + "vpc_name" => %vpc.identity.name, + "start_vni" => ?vni, + ); + } + } + } + + // We've failed to find a VNI after searching the entire range, so we'll + // return a 503 at this point. + error!( + opctx.log, + "failed to find a VNI after searching entire range"; + ); + Err(Error::unavail("Failed to find a free VNI for this VPC")) } + // Internal implementation for creating a VPC. + // + // This returns an optional VPC. If it is None, then we failed to insert a + // VPC specifically because there are no available VNIs. All other errors + // are returned in the `Result::Err` variant. async fn project_create_vpc_raw( &self, opctx: &OpContext, authz_project: &authz::Project, vpc_query: InsertVpcQuery, - ) -> Result<(authz::Vpc, Vpc), Error> { + ) -> Result, Error> { use db::schema::vpc::dsl; assert_eq!(authz_project.id(), vpc_query.vpc.project_id); @@ -312,30 +363,48 @@ impl DataStore { let project_id = vpc_query.vpc.project_id; let conn = self.pool_connection_authorized(opctx).await?; - let vpc: Vpc = Project::insert_resource( + let result: Result = Project::insert_resource( project_id, diesel::insert_into(dsl::vpc).values(vpc_query), ) .insert_and_get_result_async(&conn) - .await - .map_err(|e| match e { - AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { - type_name: ResourceType::Project, - lookup_type: LookupType::ById(project_id), - }, - AsyncInsertError::DatabaseError(e) => public_error_from_diesel( - e, - ErrorHandler::Conflict(ResourceType::Vpc, name.as_str()), - ), - })?; - Ok(( - authz::Vpc::new( - authz_project.clone(), - vpc.id(), - LookupType::ByName(vpc.name().to_string()), - ), - vpc, - )) + .await; + match result { + Ok(vpc) => Ok(Some(( + authz::Vpc::new( + authz_project.clone(), + vpc.id(), + LookupType::ByName(vpc.name().to_string()), + ), + vpc, + ))), + Err(AsyncInsertError::CollectionNotFound) => { + Err(Error::ObjectNotFound { + type_name: ResourceType::Project, + lookup_type: LookupType::ById(project_id), + }) + } + Err(AsyncInsertError::DatabaseError( + DieselError::DatabaseError( + DatabaseErrorKind::NotNullViolation, + info, + ), + )) if info + .message() + .starts_with("null value in column \"vni\"") => + { + // We failed the non-null check on the VNI column, which means + // we could not find a valid VNI in our search range. Return + // None instead to signal the error. + Ok(None) + } + Err(AsyncInsertError::DatabaseError(e)) => { + Err(public_error_from_diesel( + e, + ErrorHandler::Conflict(ResourceType::Vpc, name.as_str()), + )) + } + } } pub async fn project_update_vpc( @@ -1092,3 +1161,234 @@ impl DataStore { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::datastore::datastore_test; + use crate::db::model::Project; + use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; + use nexus_test_utils::db::test_setup_database; + use nexus_types::external_api::params; + use omicron_common::api::external; + use omicron_test_utils::dev; + use slog::info; + + // Test that we detect the right error condition and return None when we + // fail to insert a VPC due to VNI exhaustion. + // + // This is a bit awkward, but we'll test this by inserting a bunch of VPCs, + // and checking that we get the expected error response back from the + // `project_create_vpc_raw` call. + #[tokio::test] + async fn test_project_create_vpc_raw_returns_none_on_vni_exhaustion() { + usdt::register_probes().unwrap(); + let logctx = dev::test_setup_log( + "test_project_create_vpc_raw_returns_none_on_vni_exhaustion", + ); + let log = &logctx.log; + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // Create a project. + let project_params = params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: String::from("test project"), + }, + }; + let project = Project::new(Uuid::new_v4(), project_params); + let (authz_project, _) = datastore + .project_create(&opctx, project) + .await + .expect("failed to create project"); + + let starting_vni = 2048; + let description = String::from("test vpc"); + for vni in 0..=MAX_VNI_SEARCH_RANGE_SIZE { + // Create an incomplete VPC and make sure it has the next available + // VNI. + let name: external::Name = format!("vpc{vni}").parse().unwrap(); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = + Vni(external::Vni::try_from(starting_vni + vni).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating initial VPC"; + "index" => vni, + "vni" => ?this_vni, + ); + let query = InsertVpcQuery::new(incomplete_vpc); + let (_, db_vpc) = datastore + .project_create_vpc_raw(&opctx, &authz_project, query) + .await + .expect("failed to create initial set of VPCs") + .expect("expected an actual VPC"); + info!( + log, + "created VPC"; + "vpc" => ?db_vpc, + ); + } + + // At this point, we've filled all the VNIs starting from 2048. Let's + // try to allocate one more, also starting from that position. This + // should fail, because we've explicitly filled the entire range we'll + // search above. + let name: external::Name = "dead-vpc".parse().unwrap(); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = Vni(external::Vni::try_from(starting_vni).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating VPC when all VNIs are allocated"; + "vni" => ?this_vni, + ); + let query = InsertVpcQuery::new(incomplete_vpc); + let Ok(None) = datastore + .project_create_vpc_raw(&opctx, &authz_project, query) + .await + else { + panic!("Expected Ok(None) when creating a VPC without any available VNIs"); + }; + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + + // Test that we appropriately retry when there are no available VNIs. + // + // This is a bit awkward, but we'll test this by inserting a bunch of VPCs, + // and then check that we correctly retry + #[tokio::test] + async fn test_project_create_vpc_retries() { + usdt::register_probes().unwrap(); + let logctx = dev::test_setup_log("test_project_create_vpc_retries"); + let log = &logctx.log; + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // Create a project. + let project_params = params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: String::from("test project"), + }, + }; + let project = Project::new(Uuid::new_v4(), project_params); + let (authz_project, _) = datastore + .project_create(&opctx, project) + .await + .expect("failed to create project"); + + let starting_vni = 2048; + let description = String::from("test vpc"); + for vni in 0..=MAX_VNI_SEARCH_RANGE_SIZE { + // Create an incomplete VPC and make sure it has the next available + // VNI. + let name: external::Name = format!("vpc{vni}").parse().unwrap(); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = + Vni(external::Vni::try_from(starting_vni + vni).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating initial VPC"; + "index" => vni, + "vni" => ?this_vni, + ); + let query = InsertVpcQuery::new(incomplete_vpc); + let (_, db_vpc) = datastore + .project_create_vpc_raw(&opctx, &authz_project, query) + .await + .expect("failed to create initial set of VPCs") + .expect("expected an actual VPC"); + info!( + log, + "created VPC"; + "vpc" => ?db_vpc, + ); + } + + // Similar to the above test, we've fill all available VPCs starting at + // `starting_vni`. Let's attempt to allocate one beginning there, which + // _should_ fail and be internally retried. Note that we're using + // `project_create_vpc()` here instead of the raw version, to check that + // retry logic. + let name: external::Name = "dead-at-first-vpc".parse().unwrap(); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = Vni(external::Vni::try_from(starting_vni).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating VPC when all VNIs are allocated"; + "vni" => ?this_vni, + ); + match datastore + .project_create_vpc(&opctx, &authz_project, incomplete_vpc.clone()) + .await + { + Ok((_, vpc)) => { + assert_eq!(vpc.id(), incomplete_vpc.identity.id); + let expected_vni = starting_vni + MAX_VNI_SEARCH_RANGE_SIZE + 1; + assert_eq!(u32::from(vpc.vni.0), expected_vni); + info!(log, "successfully created VPC after retries"; "vpc" => ?vpc); + } + Err(e) => panic!("Unexpected error when inserting VPC: {e}"), + }; + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/queries/vpc.rs b/nexus/db-queries/src/db/queries/vpc.rs index b1ac8fe1e1..c29a51adb0 100644 --- a/nexus/db-queries/src/db/queries/vpc.rs +++ b/nexus/db-queries/src/db/queries/vpc.rs @@ -245,15 +245,7 @@ struct NextVni { impl NextVni { fn new(vni: Vni) -> Self { - let base_u32 = u32::from(vni.0); - // The valid range is [0, 1 << 24], so the maximum shift is whatever - // gets us to 1 << 24, and the minimum is whatever gets us back to the - // minimum guest VNI. - let max_shift = i64::from(external::Vni::MAX_VNI - base_u32); - let min_shift = i64::from( - -i32::try_from(base_u32 - external::Vni::MIN_GUEST_VNI) - .expect("Expected a valid VNI at this point"), - ); + let VniShifts { min_shift, max_shift } = VniShifts::new(vni); let generator = DefaultShiftGenerator { base: vni, max_shift, min_shift }; let inner = NextItem::new_unscoped(generator); @@ -278,3 +270,208 @@ impl NextVni { } delegate_query_fragment_impl!(NextVni); + +// Helper type to compute the shift for a `NextItem` query to find VNIs. +#[derive(Clone, Copy, Debug, PartialEq)] +struct VniShifts { + // The minimum `ShiftGenerator` shift. + min_shift: i64, + // The maximum `ShiftGenerator` shift. + max_shift: i64, +} + +/// Restrict the search for a VNI to a small range. +/// +/// VNIs are pretty sparsely allocated (the number of VPCs), and the range is +/// quite large (24 bits). To avoid memory issues, we'll restrict a search +/// for an available VNI to a small range starting from the random starting +/// VNI. +// +// NOTE: This is very small for tests, to ensure we can accurately test the +// failure mode where there are no available VNIs. +#[cfg(not(test))] +pub const MAX_VNI_SEARCH_RANGE_SIZE: u32 = 2048; +#[cfg(test)] +pub const MAX_VNI_SEARCH_RANGE_SIZE: u32 = 10; + +// Ensure that we cannot search a range that extends beyond the valid guest VNI +// range. +static_assertions::const_assert!( + MAX_VNI_SEARCH_RANGE_SIZE + <= (external::Vni::MAX_VNI - external::Vni::MIN_GUEST_VNI) +); + +impl VniShifts { + fn new(vni: Vni) -> Self { + let base_u32 = u32::from(vni.0); + let range_end = base_u32 + MAX_VNI_SEARCH_RANGE_SIZE; + + // Clamp the maximum shift at the distance to the maximum allowed VNI, + // or the maximum of the range. + let max_shift = i64::from( + (external::Vni::MAX_VNI - base_u32).min(MAX_VNI_SEARCH_RANGE_SIZE), + ); + + // And any remaining part of the range wraps around starting at the + // beginning. + let min_shift = -i64::from( + range_end.checked_sub(external::Vni::MAX_VNI).unwrap_or(0), + ); + Self { min_shift, max_shift } + } +} + +/// An iterator yielding sequential starting VNIs. +/// +/// The VPC insertion query requires a search for the next available VNI, using +/// the `NextItem` query. We limit the search for each query to avoid memory +/// issues on any one query. If we fail to find a VNI, we need to search the +/// next range. This iterator yields the starting positions for the `NextItem` +/// query, so that the entire range can be search in chunks until a free VNI is +/// found. +// +// NOTE: It's technically possible for this to lead to searching the very +// initial portion of the range twice. If we end up wrapping around so that the +// last position yielded by this iterator is `start - x`, then we'll end up +// searching from `start - x` to `start + (MAX_VNI_SEARCH_RANGE_SIZE - x)`, and +// so search those first few after `start` again. This is both innocuous and +// really unlikely. +#[derive(Clone, Copy, Debug)] +pub struct VniSearchIter { + start: u32, + current: u32, + has_wrapped: bool, +} + +impl VniSearchIter { + pub const STEP_SIZE: u32 = MAX_VNI_SEARCH_RANGE_SIZE; + + /// Create a search range, starting from the provided VNI. + pub fn new(start: external::Vni) -> Self { + let start = u32::from(start); + Self { start, current: start, has_wrapped: false } + } +} + +impl std::iter::Iterator for VniSearchIter { + type Item = external::Vni; + + fn next(&mut self) -> Option { + // If we've wrapped around and the computed position is beyond where we + // started, then the ite + if self.has_wrapped && self.current > self.start { + return None; + } + + // Compute the next position. + // + // Make sure we wrap around to the mininum guest VNI. Note that we + // consider the end of the range inclusively, so we subtract one in the + // offset below to end up _at_ the min guest VNI. + let mut next = self.current + MAX_VNI_SEARCH_RANGE_SIZE; + if next > external::Vni::MAX_VNI { + next -= external::Vni::MAX_VNI; + next += external::Vni::MIN_GUEST_VNI - 1; + self.has_wrapped = true; + } + let current = self.current; + self.current = next; + Some(external::Vni::try_from(current).unwrap()) + } +} + +#[cfg(test)] +mod tests { + use super::external; + use super::Vni; + use super::VniSearchIter; + use super::VniShifts; + use super::MAX_VNI_SEARCH_RANGE_SIZE; + + // Ensure that when the search range lies entirely within the range of VNIs, + // we search from the start VNI through the maximum allowed range size. + #[test] + fn test_vni_shift_no_wrapping() { + let vni = Vni(external::Vni::try_from(2048).unwrap()); + let VniShifts { min_shift, max_shift } = VniShifts::new(vni); + assert_eq!(min_shift, 0); + assert_eq!(max_shift, i64::from(MAX_VNI_SEARCH_RANGE_SIZE)); + assert_eq!(max_shift - min_shift, i64::from(MAX_VNI_SEARCH_RANGE_SIZE)); + } + + // Ensure that we wrap correctly, when the starting VNI happens to land + // quite close to the end of the allowed range. + #[test] + fn test_vni_shift_with_wrapping() { + let offset = 5; + let vni = + Vni(external::Vni::try_from(external::Vni::MAX_VNI - offset) + .unwrap()); + let VniShifts { min_shift, max_shift } = VniShifts::new(vni); + assert_eq!(min_shift, -i64::from(MAX_VNI_SEARCH_RANGE_SIZE - offset)); + assert_eq!(max_shift, i64::from(offset)); + assert_eq!(max_shift - min_shift, i64::from(MAX_VNI_SEARCH_RANGE_SIZE)); + } + + #[test] + fn test_vni_search_iter_steps() { + let start = external::Vni::try_from(2048).unwrap(); + let mut it = VniSearchIter::new(start); + let next = it.next().unwrap(); + assert_eq!(next, start); + let next = it.next().unwrap(); + assert_eq!( + next, + external::Vni::try_from( + u32::from(start) + MAX_VNI_SEARCH_RANGE_SIZE + ) + .unwrap() + ); + } + + #[test] + fn test_vni_search_iter_full_count() { + let start = + external::Vni::try_from(external::Vni::MIN_GUEST_VNI).unwrap(); + + let last = VniSearchIter::new(start).last().unwrap(); + println!("{:?}", last); + + pub const fn div_ceil(x: u32, y: u32) -> u32 { + let d = x / y; + let r = x % y; + if r > 0 && y > 0 { + d + 1 + } else { + d + } + } + const N_EXPECTED: u32 = div_ceil( + external::Vni::MAX_VNI - external::Vni::MIN_GUEST_VNI, + MAX_VNI_SEARCH_RANGE_SIZE, + ); + let count = u32::try_from(VniSearchIter::new(start).count()).unwrap(); + assert_eq!(count, N_EXPECTED); + } + + #[test] + fn test_vni_search_iter_wrapping() { + // Start from just before the end of the range. + let start = + external::Vni::try_from(external::Vni::MAX_VNI - 1).unwrap(); + let mut it = VniSearchIter::new(start); + + // We should yield that start position first. + let next = it.next().unwrap(); + assert_eq!(next, start); + + // The next value should be wrapped around to the beginning. + // + // Subtract 2 because we _include_ the max VNI in the search range. + let next = it.next().unwrap(); + assert_eq!( + u32::from(next), + external::Vni::MIN_GUEST_VNI + MAX_VNI_SEARCH_RANGE_SIZE - 2 + ); + } +} diff --git a/nexus/db-queries/src/lib.rs b/nexus/db-queries/src/lib.rs index 29c33039ff..a693f7ff42 100644 --- a/nexus/db-queries/src/lib.rs +++ b/nexus/db-queries/src/lib.rs @@ -17,3 +17,22 @@ extern crate newtype_derive; #[cfg(test)] #[macro_use] extern crate diesel; + +#[usdt::provider(provider = "nexus__db__queries")] +mod probes { + // Fires before we start a search over a range for a VNI. + // + // Includes the starting VNI and the size of the range being searched. + fn vni__search__range__start( + _: &usdt::UniqueId, + start_vni: u32, + size: u32, + ) { + } + + // Fires when we successfully find a VNI. + fn vni__search__range__found(_: &usdt::UniqueId, vni: u32) {} + + // Fires when we fail to find a VNI in the provided range. + fn vni__search__range__empty(_: &usdt::UniqueId) {} +} From 825e4f39c8efb147a108d13d87def0221c27c8df Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Sat, 21 Oct 2023 16:12:26 -0400 Subject: [PATCH 74/89] Make sure project_delete_snapshot does its job (#4308) `project_delete_snapshot`'s job is to soft-delete snapshots but due to incorrectly using `check_if_exists` with `execute_async`, the state a snapshot was in was not being validated. This was causing a snapshot delete request to proceed for a snapshot that was in state Creating, causing a case where a volume was hard deleted before it had its associated resource accounting performed, leading to regions associated with deleted resources that would never be cleaned up. Fix this and ensure expected behaviour via a test. --- nexus/db-queries/src/db/datastore/snapshot.rs | 67 ++++++++++++---- nexus/tests/integration_tests/disks.rs | 78 +++++++++++++++++++ nexus/tests/integration_tests/snapshots.rs | 32 +++++++- 3 files changed, 163 insertions(+), 14 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/snapshot.rs b/nexus/db-queries/src/db/datastore/snapshot.rs index 59fb00c84d..7c03e4bd40 100644 --- a/nexus/db-queries/src/db/datastore/snapshot.rs +++ b/nexus/db-queries/src/db/datastore/snapshot.rs @@ -19,6 +19,7 @@ use crate::db::model::Snapshot; use crate::db::model::SnapshotState; use crate::db::pagination::paginated; use crate::db::update_and_check::UpdateAndCheck; +use crate::db::update_and_check::UpdateStatus; use crate::db::TransactionError; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; @@ -252,30 +253,70 @@ impl DataStore { use db::schema::snapshot::dsl; - let updated_rows = diesel::update(dsl::snapshot) + let result = diesel::update(dsl::snapshot) .filter(dsl::time_deleted.is_null()) .filter(dsl::gen.eq(gen)) .filter(dsl::id.eq(snapshot_id)) - .filter(dsl::state.eq_any(ok_to_delete_states)) + .filter(dsl::state.eq_any(ok_to_delete_states.clone())) .set(( dsl::time_deleted.eq(now), dsl::state.eq(SnapshotState::Destroyed), )) .check_if_exists::(snapshot_id) - .execute_async(&*self.pool_connection_authorized(&opctx).await?) + .execute_and_check(&*self.pool_connection_authorized(&opctx).await?) .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Snapshot, + LookupType::ById(snapshot_id), + ), + ) + })?; + + match result.status { + UpdateStatus::Updated => { + // snapshot was soft deleted ok + Ok(result.found.id()) + } - if updated_rows == 0 { - // Either: - // - // - the snapshot was already deleted - // - the generation number changed - // - the state of the snapshot isn't one of `ok_to_delete_states` + UpdateStatus::NotUpdatedButExists => { + let snapshot = result.found; - return Err(Error::invalid_request("snapshot cannot be deleted")); - } + // if the snapshot was already deleted, return Ok - this + // function must remain idempotent for the same input. + if snapshot.time_deleted().is_some() + && snapshot.state == SnapshotState::Destroyed + { + Ok(snapshot.id()) + } else { + // if the snapshot was not deleted, figure out why + if !ok_to_delete_states.contains(&snapshot.state) { + Err(Error::invalid_request(&format!( + "snapshot cannot be deleted in state {:?}", + snapshot.state, + ))) + } else if snapshot.gen != gen { + Err(Error::invalid_request(&format!( + "snapshot cannot be deleted: mismatched generation {:?} != {:?}", + gen, + snapshot.gen, + ))) + } else { + error!( + opctx.log, + "snapshot exists but cannot be deleted: {:?} (db_snapshot is {:?}", + snapshot, + db_snapshot, + ); - Ok(snapshot_id) + Err(Error::invalid_request( + "snapshot exists but cannot be deleted", + )) + } + } + } + } } } diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index 71a3977192..a5a8339c34 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -1672,6 +1672,84 @@ async fn test_disk_create_for_importing(cptestctx: &ControlPlaneTestContext) { disks_eq(&disks[0], &disk); } +#[nexus_test] +async fn test_project_delete_disk_no_auth_idempotent( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + DiskTest::new(&cptestctx).await; + create_org_and_project(client).await; + + // Create a disk + let disks_url = get_disks_url(); + + let new_disk = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: DISK_NAME.parse().unwrap(), + description: String::from("sells rainsticks"), + }, + disk_source: params::DiskSource::Blank { + block_size: params::BlockSize::try_from(512).unwrap(), + }, + size: ByteCount::from_gibibytes_u32(1), + }; + + NexusRequest::new( + RequestBuilder::new(client, Method::POST, &disks_url) + .body(Some(&new_disk)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body::() + .unwrap(); + + let disk_url = get_disk_url(DISK_NAME); + let disk = disk_get(&client, &disk_url).await; + assert_eq!(disk.state, DiskState::Detached); + + // Call project_delete_disk_no_auth twice, ensuring that the disk is either + // there before deleting and not afterwards. + + let nexus = &cptestctx.server.apictx().nexus; + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + let (.., db_disk) = LookupPath::new(&opctx, &datastore) + .disk_id(disk.identity.id) + .fetch() + .await + .unwrap(); + + assert_eq!(db_disk.id(), disk.identity.id); + + datastore + .project_delete_disk_no_auth( + &disk.identity.id, + &[DiskState::Detached, DiskState::Faulted], + ) + .await + .unwrap(); + + let r = LookupPath::new(&opctx, &datastore) + .disk_id(disk.identity.id) + .fetch() + .await; + + assert!(r.is_err()); + + datastore + .project_delete_disk_no_auth( + &disk.identity.id, + &[DiskState::Detached, DiskState::Faulted], + ) + .await + .unwrap(); +} + async fn disk_get(client: &ClientTestContext, disk_url: &str) -> Disk { NexusRequest::object_get(client, disk_url) .authn_as(AuthnMode::PrivilegedUser) diff --git a/nexus/tests/integration_tests/snapshots.rs b/nexus/tests/integration_tests/snapshots.rs index b3cf4bb594..1dd32e6769 100644 --- a/nexus/tests/integration_tests/snapshots.rs +++ b/nexus/tests/integration_tests/snapshots.rs @@ -1041,7 +1041,25 @@ async fn test_create_snapshot_record_idempotent( external::Error::ObjectAlreadyExists { .. }, )); - // Test project_delete_snapshot is idempotent + // Move snapshot from Creating to Ready + + let (.., authz_snapshot, db_snapshot) = LookupPath::new(&opctx, &datastore) + .snapshot_id(snapshot_created_1.id()) + .fetch_for(authz::Action::Modify) + .await + .unwrap(); + + datastore + .project_snapshot_update_state( + &opctx, + &authz_snapshot, + db_snapshot.gen, + db::model::SnapshotState::Ready, + ) + .await + .unwrap(); + + // Grab the new snapshot (so generation number is updated) let (.., authz_snapshot, db_snapshot) = LookupPath::new(&opctx, &datastore) .snapshot_id(snapshot_created_1.id()) @@ -1049,6 +1067,8 @@ async fn test_create_snapshot_record_idempotent( .await .unwrap(); + // Test project_delete_snapshot is idempotent for the same input + datastore .project_delete_snapshot( &opctx, @@ -1064,6 +1084,16 @@ async fn test_create_snapshot_record_idempotent( .await .unwrap(); + { + // Ensure the snapshot is gone + let r = LookupPath::new(&opctx, &datastore) + .snapshot_id(snapshot_created_1.id()) + .fetch_for(authz::Action::Read) + .await; + + assert!(r.is_err()); + } + datastore .project_delete_snapshot( &opctx, From 1cd3314e234533343b63e0d4f9b71a8ce2e4c432 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Sat, 21 Oct 2023 13:14:15 -0700 Subject: [PATCH 75/89] Bump SP versions to 1.0.3 (#4305) --- tools/hubris_checksums | 14 +++++++------- tools/hubris_version | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/hubris_checksums b/tools/hubris_checksums index e2b16d345e..1396af4d60 100644 --- a/tools/hubris_checksums +++ b/tools/hubris_checksums @@ -1,7 +1,7 @@ -204328b941deab8bfd3d6e34af96ef04489672fa3b0d5419b60456f9b052e789 build-gimlet-c-image-default-v1.0.2.zip -0ebaa9d98c3734420a482160a7a19dd09510ea3bdc573a090a97ec47137bd624 build-gimlet-d-image-default-v1.0.2.zip -39ec8fd0c946b744e59d9c1b89914f354c60f54e974398029d1dea9d31681f05 build-gimlet-e-image-default-v1.0.2.zip -fa5dc36a7a70eeb45d4c4b3b314ba54ee820b3d57ffc07defcc3ae07c142329c build-psc-b-image-default-v1.0.2.zip -4a9850100f8b5fcbbdd11d41ccd8d5037059697a9b65c1ba2dba48a6565ba9e7 build-psc-c-image-default-v1.0.2.zip -1bb870d7921c1731ec83dc38b8e3425703ec17efa614d75e92f07a551312f54b build-sidecar-b-image-default-v1.0.2.zip -6aed0e15e0025bb87a22ecea60d75fa71b54b83bea1e213b8cd5bdb02e7ccb2d build-sidecar-c-image-default-v1.0.2.zip +2df01d7dd17423588c99de4361694efdb6bd375e2f54db053320eeead3e07eda build-gimlet-c-image-default-v1.0.3.zip +8ac0eb6d7817825c6318feb8327f5758a33ccd2479512e3e2424f0eb8e290010 build-gimlet-d-image-default-v1.0.3.zip +eeeb72ec81a843fa1f5093096d1e4500aba6ce01c2d21040a2a10a092595d945 build-gimlet-e-image-default-v1.0.3.zip +de0d9028929322f6d5afc4cb52c198b3402c93a38aa15f9d378617ca1d1112c9 build-psc-b-image-default-v1.0.3.zip +11a6235d852bd75548f12d85b0913cb4ccb0aff3c38bf8a92510a2b9c14dad3c build-psc-c-image-default-v1.0.3.zip +3f863d46a462432f19d3fb5a293b8106da6e138de80271f869692bd29abd994b build-sidecar-b-image-default-v1.0.3.zip +2a9feac7f2da61b843d00edf2693c31c118f202c6cd889d1d1758ea1dd95dbca build-sidecar-c-image-default-v1.0.3.zip diff --git a/tools/hubris_version b/tools/hubris_version index 0f1729d791..b00c3286fe 100644 --- a/tools/hubris_version +++ b/tools/hubris_version @@ -1 +1 @@ -TAGS=(gimlet-v1.0.2 psc-v1.0.2 sidecar-v1.0.2) +TAGS=(gimlet-v1.0.3 psc-v1.0.3 sidecar-v1.0.3) From 51b6b160c4913ee77685ab95dbfc50047d5503fe Mon Sep 17 00:00:00 2001 From: Andy Fiddaman Date: Mon, 23 Oct 2023 18:06:41 +0100 Subject: [PATCH 76/89] Combine host and trampoline OS images into one CI job (#4273) There are two OS build jobs that each check out and build many repositories in order to build a host image. By combining them into one job, we only pay the cost of this checkout and build phase once. It doesn't actually save a lot of wall time because the two jobs run in parallel, but it saves starting a VM and about 20 minutes of compute time. --- .github/buildomat/jobs/host-image.sh | 48 ++++++++++++++++--- .github/buildomat/jobs/trampoline-image.sh | 54 ---------------------- .github/buildomat/jobs/tuf-repo.sh | 7 +-- tools/build-host-image.sh | 16 ------- 4 files changed, 44 insertions(+), 81 deletions(-) delete mode 100755 .github/buildomat/jobs/trampoline-image.sh diff --git a/.github/buildomat/jobs/host-image.sh b/.github/buildomat/jobs/host-image.sh index ba0b4e1ac3..2f4d146a48 100755 --- a/.github/buildomat/jobs/host-image.sh +++ b/.github/buildomat/jobs/host-image.sh @@ -1,11 +1,12 @@ #!/bin/bash #: -#: name = "helios / build OS image" +#: name = "helios / build OS images" #: variety = "basic" #: target = "helios-2.0" #: rust_toolchain = "1.72.1" #: output_rules = [ -#: "=/work/helios/image/output/os.tar.gz", +#: "=/work/helios/upload/os-host.tar.gz", +#: "=/work/helios/upload/os-trampoline.tar.gz", #: ] #: access_repos = [ #: "oxidecomputer/amd-apcb", @@ -44,14 +45,49 @@ TOP=$PWD source "$TOP/tools/include/force-git-over-https.sh" -# Checkout helios at a pinned commit into /work/helios -git clone https://github.com/oxidecomputer/helios.git /work/helios -cd /work/helios +# Check out helios into /work/helios +HELIOSDIR=/work/helios +git clone https://github.com/oxidecomputer/helios.git "$HELIOSDIR" +cd "$HELIOSDIR" +# Record the branch and commit in the output +git status --branch --porcelain=2 +# Setting BUILD_OS to no makes setup skip repositories we don't need for +# building the OS itself (we are just building an image from already built OS). +BUILD_OS=no gmake setup + +# Commands that "helios-build" would ask us to run (either explicitly or +# implicitly, to avoid an error). +rc=0 +pfexec pkg install -q /system/zones/brand/omicron1/tools || rc=$? +case $rc in + # `man pkg` notes that exit code 4 means no changes were made because + # there is nothing to do; that's fine. Any other exit code is an error. + 0 | 4) ;; + *) exit $rc ;; +esac + +pfexec zfs create -p "rpool/images/$USER" + # TODO: Consider importing zones here too? cd "$TOP" +OUTPUTDIR="$HELIOSDIR/upload" +mkdir "$OUTPUTDIR" + +banner OS ./tools/build-host-image.sh -B \ -S /input/package/work/zones/switch-asic.tar.gz \ - /work/helios \ + "$HELIOSDIR" \ /input/package/work/global-zone-packages.tar.gz + +mv "$HELIOSDIR/image/output/os.tar.gz" "$OUTPUTDIR/os-host.tar.gz" + +banner Trampoline + +./tools/build-host-image.sh -R \ + "$HELIOSDIR" \ + /input/package/work/trampoline-global-zone-packages.tar.gz + +mv "$HELIOSDIR/image/output/os.tar.gz" "$OUTPUTDIR/os-trampoline.tar.gz" + diff --git a/.github/buildomat/jobs/trampoline-image.sh b/.github/buildomat/jobs/trampoline-image.sh deleted file mode 100755 index 6014d7dca0..0000000000 --- a/.github/buildomat/jobs/trampoline-image.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -#: -#: name = "helios / build trampoline OS image" -#: variety = "basic" -#: target = "helios-2.0" -#: rust_toolchain = "1.72.1" -#: output_rules = [ -#: "=/work/helios/image/output/os.tar.gz", -#: ] -#: access_repos = [ -#: "oxidecomputer/amd-apcb", -#: "oxidecomputer/amd-efs", -#: "oxidecomputer/amd-firmware", -#: "oxidecomputer/amd-flash", -#: "oxidecomputer/amd-host-image-builder", -#: "oxidecomputer/boot-image-tools", -#: "oxidecomputer/chelsio-t6-roms", -#: "oxidecomputer/compliance-pilot", -#: "oxidecomputer/facade", -#: "oxidecomputer/helios", -#: "oxidecomputer/helios-omicron-brand", -#: "oxidecomputer/helios-omnios-build", -#: "oxidecomputer/helios-omnios-extra", -#: "oxidecomputer/nanobl-rs", -#: ] -#: -#: [dependencies.package] -#: job = "helios / package" -#: -#: [[publish]] -#: series = "image" -#: name = "os-trampoline.tar.gz" -#: from_output = "/work/helios/image/output/os.tar.gz" -#: - -set -o errexit -set -o pipefail -set -o xtrace - -cargo --version -rustc --version - -TOP=$PWD - -source "$TOP/tools/include/force-git-over-https.sh" - -# Checkout helios at a pinned commit into /work/helios -git clone https://github.com/oxidecomputer/helios.git /work/helios -cd /work/helios - -cd "$TOP" -./tools/build-host-image.sh -R \ - /work/helios \ - /input/package/work/trampoline-global-zone-packages.tar.gz diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index ea25ab5834..29cf7fa85e 100644 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -19,10 +19,7 @@ #: job = "helios / package" #: #: [dependencies.host] -#: job = "helios / build OS image" -#: -#: [dependencies.trampoline] -#: job = "helios / build trampoline OS image" +#: job = "helios / build OS images" #: #: [[publish]] #: series = "rot-all" @@ -139,7 +136,7 @@ name = "$kind" version = "$VERSION" [artifact.$kind.source] kind = "file" -path = "/input/$kind/work/helios/image/output/os.tar.gz" +path = "/input/host/work/helios/upload/os-$kind.tar.gz" EOF done diff --git a/tools/build-host-image.sh b/tools/build-host-image.sh index d492e84b81..c194edb603 100755 --- a/tools/build-host-image.sh +++ b/tools/build-host-image.sh @@ -92,22 +92,6 @@ function main # Move to the helios checkout cd "$HELIOS_PATH" - # Create the "./helios-build" command, which lets us build images - gmake setup - - # Commands that "./helios-build" would ask us to run (either explicitly - # or implicitly, to avoid an error). - rc=0 - pfexec pkg install -q /system/zones/brand/omicron1/tools || rc=$? - case $rc in - # `man pkg` notes that exit code 4 means no changes were made because - # there is nothing to do; that's fine. Any other exit code is an error. - 0 | 4) ;; - *) exit $rc ;; - esac - - pfexec zfs create -p rpool/images/"$USER" - HELIOS_REPO=https://pkg.oxide.computer/helios/2/dev/ # Build an image name that includes the omicron and host OS hashes From 3af87d2e29a9bf1a75f86a2f472cf8a0ec719737 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Mon, 23 Oct 2023 15:08:28 -0500 Subject: [PATCH 77/89] Bump web console (utilization table) (#4318) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/oxidecomputer/console/compare/3538c32a...bd65b9da * [bd65b9da](https://github.com/oxidecomputer/console/commit/bd65b9da) loading state for utilization table, use QueryParamTabs * [8e74accf](https://github.com/oxidecomputer/console/commit/8e74accf) handle empty metrics list on silo utilization * [672cb208](https://github.com/oxidecomputer/console/commit/672cb208) bump API spec (BGP endpoints, not used yet) * [a50777c5](https://github.com/oxidecomputer/console/commit/a50777c5) oxidecomputer/console#1795 * [8978e07a](https://github.com/oxidecomputer/console/commit/8978e07a) bump omicron version (no changes) * [992fb1e1](https://github.com/oxidecomputer/console/commit/992fb1e1) oxidecomputer/console#1797 * [e5b8d029](https://github.com/oxidecomputer/console/commit/e5b8d029) remove unnecessary @types/testing-library__jest-dom dep * [e9a78617](https://github.com/oxidecomputer/console/commit/e9a78617) tanstack query 5 went stable --- tools/console_version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/console_version b/tools/console_version index 218aef576d..7e8d352efd 100644 --- a/tools/console_version +++ b/tools/console_version @@ -1,2 +1,2 @@ -COMMIT="3538c32a5189bd22df8f6a573399dacfbe81efaa" -SHA2="3289989f2cd6c71ea800e73231190455cc8e4e45ae9304293050b925a9fd9423" +COMMIT="bd65b9da7019ad812dd056e7fc182df2cf4ec128" +SHA2="e4d4f33996a6e89b972fac61737acb7f1dbd21943d1f6bef776d4ee9bcccd2b0" From 35034f75d81821bf7f7901485143ff5f72608e9f Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Mon, 23 Oct 2023 18:58:58 -0700 Subject: [PATCH 78/89] Re-introduce support for setting a VLAN for an uplink from RSS (#4319) Going from the previous `UplinkConfig` format to the `PortConfigV1` introduced with the BGP work seems to have lost the ability to configure the VLAN ID for reaching a gateway. The actual APIs and functionality is still there so just needed to be wired back up. --- common/src/api/internal/shared.rs | 3 + nexus/src/app/rack.rs | 1 + .../app/sagas/switch_port_settings_apply.rs | 6 +- nexus/tests/integration_tests/switch_port.rs | 31 ++++++--- openapi/bootstrap-agent.json | 7 +++ openapi/nexus-internal.json | 7 +++ openapi/sled-agent.json | 7 +++ openapi/wicketd.json | 7 +++ schema/rss-sled-plan.json | 9 +++ sled-agent/src/bootstrap/early_networking.rs | 63 ++++++++++++++++++- sled-agent/src/rack_setup/service.rs | 1 + wicket/src/rack_setup/config_template.toml | 3 + wicket/src/rack_setup/config_toml.rs | 23 +++++-- wicket/src/ui/panes/rack_setup.rs | 7 ++- wicketd/src/rss_config.rs | 1 + 15 files changed, 161 insertions(+), 15 deletions(-) diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 784da8fcc6..971dbbabbf 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -111,6 +111,8 @@ pub struct RouteConfig { pub destination: IpNetwork, /// The nexthop/gateway address. pub nexthop: IpAddr, + /// The VLAN ID the gateway is reachable over. + pub vid: Option, } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] @@ -137,6 +139,7 @@ impl From for PortConfigV1 { routes: vec![RouteConfig { destination: "0.0.0.0/0".parse().unwrap(), nexthop: value.gateway_ip.into(), + vid: value.uplink_vid, }], addresses: vec![value.uplink_cidr.into()], switch: value.switch, diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 907c3ffa78..21918d2687 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -663,6 +663,7 @@ impl super::Nexus { .map(|r| SledRouteConfig { destination: r.dst, nexthop: r.gw.ip(), + vid: r.vid.map(Into::into), }) .collect(), addresses: info.addresses.iter().map(|a| a.address).collect(), diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index 93dc45751a..8442080979 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -911,7 +911,11 @@ pub(crate) async fn bootstore_update( routes: settings .routes .iter() - .map(|r| RouteConfig { destination: r.dst, nexthop: r.gw.ip() }) + .map(|r| RouteConfig { + destination: r.dst, + nexthop: r.gw.ip(), + vid: r.vid.map(Into::into), + }) .collect(), addresses: settings.addresses.iter().map(|a| a.address).collect(), switch: switch_location, diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index fada45694d..d4fd10f819 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -128,11 +128,18 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { settings.routes.insert( "phy0".into(), RouteConfig { - routes: vec![Route { - dst: "1.2.3.0/24".parse().unwrap(), - gw: "1.2.3.4".parse().unwrap(), - vid: None, - }], + routes: vec![ + Route { + dst: "1.2.3.0/24".parse().unwrap(), + gw: "1.2.3.4".parse().unwrap(), + vid: None, + }, + Route { + dst: "5.6.7.0/24".parse().unwrap(), + gw: "5.6.7.8".parse().unwrap(), + vid: Some(5), + }, + ], }, ); // addresses @@ -159,7 +166,7 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .unwrap(); assert_eq!(created.links.len(), 1); - assert_eq!(created.routes.len(), 1); + assert_eq!(created.routes.len(), 2); assert_eq!(created.addresses.len(), 1); let link0 = &created.links[0]; @@ -178,6 +185,11 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { let route0 = &created.routes[0]; assert_eq!(route0.dst, "1.2.3.0/24".parse().unwrap()); assert_eq!(route0.gw, "1.2.3.4".parse().unwrap()); + assert_eq!(route0.vlan_id, None); + let route1 = &created.routes[1]; + assert_eq!(route1.dst, "5.6.7.0/24".parse().unwrap()); + assert_eq!(route1.gw, "5.6.7.8".parse().unwrap()); + assert_eq!(route1.vlan_id, Some(5)); let addr0 = &created.addresses[0]; assert_eq!(addr0.address, "203.0.113.10/24".parse().unwrap()); @@ -195,7 +207,7 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .unwrap(); assert_eq!(roundtrip.links.len(), 1); - assert_eq!(roundtrip.routes.len(), 1); + assert_eq!(roundtrip.routes.len(), 2); assert_eq!(roundtrip.addresses.len(), 1); let link0 = &roundtrip.links[0]; @@ -214,6 +226,11 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { let route0 = &roundtrip.routes[0]; assert_eq!(route0.dst, "1.2.3.0/24".parse().unwrap()); assert_eq!(route0.gw, "1.2.3.4".parse().unwrap()); + assert_eq!(route0.vlan_id, None); + let route1 = &roundtrip.routes[1]; + assert_eq!(route1.dst, "5.6.7.0/24".parse().unwrap()); + assert_eq!(route1.gw, "5.6.7.8".parse().unwrap()); + assert_eq!(route1.vlan_id, Some(5)); let addr0 = &roundtrip.addresses[0]; assert_eq!(addr0.address, "203.0.113.10/24".parse().unwrap()); diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 6dcf756737..0a954fff0d 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -908,6 +908,13 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" + }, + "vid": { + "nullable": true, + "description": "The VLAN ID the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 } }, "required": [ diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 411c52ddff..1a3be03de1 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -4507,6 +4507,13 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" + }, + "vid": { + "nullable": true, + "description": "The VLAN ID the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 } }, "required": [ diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 486662853c..ec070a3a0b 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -2669,6 +2669,13 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" + }, + "vid": { + "nullable": true, + "description": "The VLAN ID the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 } }, "required": [ diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 75db82e8e1..9fd05a9ca7 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -2493,6 +2493,13 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" + }, + "vid": { + "nullable": true, + "description": "The VLAN ID the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 } }, "required": [ diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 39a9a68acc..7ba74ffc0a 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -581,6 +581,15 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" + }, + "vid": { + "description": "The VLAN ID the gateway is reachable over.", + "type": [ + "integer", + "null" + ], + "format": "uint16", + "minimum": 0.0 } } }, diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 6c19080e9c..c539f6fdfd 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -509,7 +509,7 @@ impl<'a> EarlyNetworkSetup<'a> { { dpd_port_settings.v4_routes.insert( dst.to_string(), - RouteSettingsV4 { link_id: link_id.0, nexthop, vid: None }, + RouteSettingsV4 { link_id: link_id.0, nexthop, vid: r.vid }, ); } if let (IpNetwork::V6(dst), IpAddr::V6(nexthop)) = @@ -517,7 +517,7 @@ impl<'a> EarlyNetworkSetup<'a> { { dpd_port_settings.v6_routes.insert( dst.to_string(), - RouteSettingsV6 { link_id: link_id.0, nexthop, vid: None }, + RouteSettingsV6 { link_id: link_id.0, nexthop, vid: r.vid }, ); } } @@ -785,6 +785,65 @@ mod tests { routes: vec![RouteConfig { destination: "0.0.0.0/0".parse().unwrap(), nexthop: uplink.gateway_ip.into(), + vid: None, + }], + addresses: vec![uplink.uplink_cidr.into()], + switch: uplink.switch, + port: uplink.uplink_port, + uplink_port_speed: uplink.uplink_port_speed, + uplink_port_fec: uplink.uplink_port_fec, + bgp_peers: vec![], + }], + bgp: vec![], + }), + }, + }; + + assert_eq!(expected, v1); + } + + #[test] + fn serialized_early_network_config_v0_to_v1_conversion_with_vid() { + let v0 = EarlyNetworkConfigV0 { + generation: 1, + rack_subnet: Ipv6Addr::UNSPECIFIED, + ntp_servers: Vec::new(), + rack_network_config: Some(RackNetworkConfigV0 { + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + uplinks: vec![UplinkConfig { + gateway_ip: Ipv4Addr::UNSPECIFIED, + switch: SwitchLocation::Switch0, + uplink_port: "Port0".to_string(), + uplink_port_speed: PortSpeed::Speed100G, + uplink_port_fec: PortFec::None, + uplink_cidr: "192.168.0.1/16".parse().unwrap(), + uplink_vid: Some(10), + }], + }), + }; + + let v0_serialized = serde_json::to_vec(&v0).unwrap(); + let bootstore_conf = + bootstore::NetworkConfig { generation: 1, blob: v0_serialized }; + + let v1 = EarlyNetworkConfig::try_from(bootstore_conf).unwrap(); + let v0_rack_network_config = v0.rack_network_config.unwrap(); + let uplink = v0_rack_network_config.uplinks[0].clone(); + let expected = EarlyNetworkConfig { + generation: 1, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: v0.ntp_servers.clone(), + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: Ipv6Network::new(v0.rack_subnet, 56).unwrap(), + infra_ip_first: v0_rack_network_config.infra_ip_first, + infra_ip_last: v0_rack_network_config.infra_ip_last, + ports: vec![PortConfigV1 { + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: uplink.gateway_ip.into(), + vid: Some(10), }], addresses: vec![uplink.uplink_cidr.into()], switch: uplink.switch, diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 7f6469d2c0..b54a8f0ba0 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -591,6 +591,7 @@ impl ServiceInner { .map(|r| NexusTypes::RouteConfig { destination: r.destination, nexthop: r.nexthop, + vid: r.vid, }) .collect(), addresses: config.addresses.clone(), diff --git a/wicket/src/rack_setup/config_template.toml b/wicket/src/rack_setup/config_template.toml index 617b61fadc..da07f42cd4 100644 --- a/wicket/src/rack_setup/config_template.toml +++ b/wicket/src/rack_setup/config_template.toml @@ -47,6 +47,9 @@ infra_ip_last = "" [[rack_network_config.ports]] # Routes associated with this port. # { nexthop = "1.2.3.4", destination = "0.0.0.0/0" } +# Can also optionally specify a VLAN id if the next hop is reachable +# over an 802.1Q tagged L2 segment. +# { nexthop = "5.6.7.8", destination = "5.6.7.0/24", vid = 5 } routes = [] # Addresses associated with this port. diff --git a/wicket/src/rack_setup/config_toml.rs b/wicket/src/rack_setup/config_toml.rs index e087c9aa7c..9616939308 100644 --- a/wicket/src/rack_setup/config_toml.rs +++ b/wicket/src/rack_setup/config_toml.rs @@ -245,6 +245,12 @@ fn populate_network_table( r.destination.to_string(), )), ); + if let Some(vid) = r.vid { + route.insert( + "vid", + Value::Integer(Formatted::new(vid.into())), + ); + } routes.push(Value::InlineTable(route)); } uplink.insert("routes", Item::Value(Value::Array(routes))); @@ -379,6 +385,7 @@ mod tests { .map(|r| InternalRouteConfig { destination: r.destination, nexthop: r.nexthop, + vid: r.vid, }) .collect(), addresses: config.addresses.clone(), @@ -478,10 +485,18 @@ mod tests { infra_ip_last: "172.30.0.10".parse().unwrap(), ports: vec![PortConfigV1 { addresses: vec!["172.30.0.1/24".parse().unwrap()], - routes: vec![RouteConfig { - destination: "0.0.0.0/0".parse().unwrap(), - nexthop: "172.30.0.10".parse().unwrap(), - }], + routes: vec![ + RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: "172.30.0.10".parse().unwrap(), + vid: None, + }, + RouteConfig { + destination: "10.20.0.0/16".parse().unwrap(), + nexthop: "10.0.0.20".parse().unwrap(), + vid: Some(20), + }, + ], bgp_peers: vec![BgpPeerConfig { asn: 47, addr: "10.2.3.4".parse().unwrap(), diff --git a/wicket/src/ui/panes/rack_setup.rs b/wicket/src/ui/panes/rack_setup.rs index 086d01ce9d..36febe404f 100644 --- a/wicket/src/ui/panes/rack_setup.rs +++ b/wicket/src/ui/panes/rack_setup.rs @@ -718,7 +718,12 @@ fn rss_config_text<'a>( vec![ Span::styled(" • Route : ", label_style), Span::styled( - format!("{} -> {}", r.destination, r.nexthop), + format!( + "{} -> {} (vid={})", + r.destination, + r.nexthop, + r.vid.unwrap_or(0), + ), ok_style, ), ] diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index a96acc56a0..d446ed71d1 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -511,6 +511,7 @@ fn validate_rack_network_config( .map(|r| BaRouteConfig { destination: r.destination, nexthop: r.nexthop, + vid: r.vid, }) .collect(), addresses: config.addresses.clone(), From 4555620332d0c25f34141dbf069803834470963c Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Mon, 23 Oct 2023 21:57:54 -0700 Subject: [PATCH 79/89] Revert "Re-introduce support for setting a VLAN for an uplink from RSS (#4319)" (#4323) This reverts commit 35034f75d81821bf7f7901485143ff5f72608e9f. https://github.com/oxidecomputer/omicron/pull/4319#issuecomment-1776439950: Note that setting the VLAN id on routes is going away entirely. The implementation was too costly on the Tofino, and we had to remove it for another feature as it did not wind up getting used. --- common/src/api/internal/shared.rs | 3 - nexus/src/app/rack.rs | 1 - .../app/sagas/switch_port_settings_apply.rs | 6 +- nexus/tests/integration_tests/switch_port.rs | 31 +++------ openapi/bootstrap-agent.json | 7 --- openapi/nexus-internal.json | 7 --- openapi/sled-agent.json | 7 --- openapi/wicketd.json | 7 --- schema/rss-sled-plan.json | 9 --- sled-agent/src/bootstrap/early_networking.rs | 63 +------------------ sled-agent/src/rack_setup/service.rs | 1 - wicket/src/rack_setup/config_template.toml | 3 - wicket/src/rack_setup/config_toml.rs | 23 ++----- wicket/src/ui/panes/rack_setup.rs | 7 +-- wicketd/src/rss_config.rs | 1 - 15 files changed, 15 insertions(+), 161 deletions(-) diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 971dbbabbf..784da8fcc6 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -111,8 +111,6 @@ pub struct RouteConfig { pub destination: IpNetwork, /// The nexthop/gateway address. pub nexthop: IpAddr, - /// The VLAN ID the gateway is reachable over. - pub vid: Option, } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] @@ -139,7 +137,6 @@ impl From for PortConfigV1 { routes: vec![RouteConfig { destination: "0.0.0.0/0".parse().unwrap(), nexthop: value.gateway_ip.into(), - vid: value.uplink_vid, }], addresses: vec![value.uplink_cidr.into()], switch: value.switch, diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 21918d2687..907c3ffa78 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -663,7 +663,6 @@ impl super::Nexus { .map(|r| SledRouteConfig { destination: r.dst, nexthop: r.gw.ip(), - vid: r.vid.map(Into::into), }) .collect(), addresses: info.addresses.iter().map(|a| a.address).collect(), diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index 8442080979..93dc45751a 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -911,11 +911,7 @@ pub(crate) async fn bootstore_update( routes: settings .routes .iter() - .map(|r| RouteConfig { - destination: r.dst, - nexthop: r.gw.ip(), - vid: r.vid.map(Into::into), - }) + .map(|r| RouteConfig { destination: r.dst, nexthop: r.gw.ip() }) .collect(), addresses: settings.addresses.iter().map(|a| a.address).collect(), switch: switch_location, diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index d4fd10f819..fada45694d 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -128,18 +128,11 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { settings.routes.insert( "phy0".into(), RouteConfig { - routes: vec![ - Route { - dst: "1.2.3.0/24".parse().unwrap(), - gw: "1.2.3.4".parse().unwrap(), - vid: None, - }, - Route { - dst: "5.6.7.0/24".parse().unwrap(), - gw: "5.6.7.8".parse().unwrap(), - vid: Some(5), - }, - ], + routes: vec![Route { + dst: "1.2.3.0/24".parse().unwrap(), + gw: "1.2.3.4".parse().unwrap(), + vid: None, + }], }, ); // addresses @@ -166,7 +159,7 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .unwrap(); assert_eq!(created.links.len(), 1); - assert_eq!(created.routes.len(), 2); + assert_eq!(created.routes.len(), 1); assert_eq!(created.addresses.len(), 1); let link0 = &created.links[0]; @@ -185,11 +178,6 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { let route0 = &created.routes[0]; assert_eq!(route0.dst, "1.2.3.0/24".parse().unwrap()); assert_eq!(route0.gw, "1.2.3.4".parse().unwrap()); - assert_eq!(route0.vlan_id, None); - let route1 = &created.routes[1]; - assert_eq!(route1.dst, "5.6.7.0/24".parse().unwrap()); - assert_eq!(route1.gw, "5.6.7.8".parse().unwrap()); - assert_eq!(route1.vlan_id, Some(5)); let addr0 = &created.addresses[0]; assert_eq!(addr0.address, "203.0.113.10/24".parse().unwrap()); @@ -207,7 +195,7 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .unwrap(); assert_eq!(roundtrip.links.len(), 1); - assert_eq!(roundtrip.routes.len(), 2); + assert_eq!(roundtrip.routes.len(), 1); assert_eq!(roundtrip.addresses.len(), 1); let link0 = &roundtrip.links[0]; @@ -226,11 +214,6 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { let route0 = &roundtrip.routes[0]; assert_eq!(route0.dst, "1.2.3.0/24".parse().unwrap()); assert_eq!(route0.gw, "1.2.3.4".parse().unwrap()); - assert_eq!(route0.vlan_id, None); - let route1 = &roundtrip.routes[1]; - assert_eq!(route1.dst, "5.6.7.0/24".parse().unwrap()); - assert_eq!(route1.gw, "5.6.7.8".parse().unwrap()); - assert_eq!(route1.vlan_id, Some(5)); let addr0 = &roundtrip.addresses[0]; assert_eq!(addr0.address, "203.0.113.10/24".parse().unwrap()); diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 0a954fff0d..6dcf756737 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -908,13 +908,6 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" - }, - "vid": { - "nullable": true, - "description": "The VLAN ID the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 } }, "required": [ diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 1a3be03de1..411c52ddff 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -4507,13 +4507,6 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" - }, - "vid": { - "nullable": true, - "description": "The VLAN ID the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 } }, "required": [ diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index ec070a3a0b..486662853c 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -2669,13 +2669,6 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" - }, - "vid": { - "nullable": true, - "description": "The VLAN ID the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 } }, "required": [ diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 9fd05a9ca7..75db82e8e1 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -2493,13 +2493,6 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" - }, - "vid": { - "nullable": true, - "description": "The VLAN ID the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 } }, "required": [ diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 7ba74ffc0a..39a9a68acc 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -581,15 +581,6 @@ "description": "The nexthop/gateway address.", "type": "string", "format": "ip" - }, - "vid": { - "description": "The VLAN ID the gateway is reachable over.", - "type": [ - "integer", - "null" - ], - "format": "uint16", - "minimum": 0.0 } } }, diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index c539f6fdfd..6c19080e9c 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -509,7 +509,7 @@ impl<'a> EarlyNetworkSetup<'a> { { dpd_port_settings.v4_routes.insert( dst.to_string(), - RouteSettingsV4 { link_id: link_id.0, nexthop, vid: r.vid }, + RouteSettingsV4 { link_id: link_id.0, nexthop, vid: None }, ); } if let (IpNetwork::V6(dst), IpAddr::V6(nexthop)) = @@ -517,7 +517,7 @@ impl<'a> EarlyNetworkSetup<'a> { { dpd_port_settings.v6_routes.insert( dst.to_string(), - RouteSettingsV6 { link_id: link_id.0, nexthop, vid: r.vid }, + RouteSettingsV6 { link_id: link_id.0, nexthop, vid: None }, ); } } @@ -785,65 +785,6 @@ mod tests { routes: vec![RouteConfig { destination: "0.0.0.0/0".parse().unwrap(), nexthop: uplink.gateway_ip.into(), - vid: None, - }], - addresses: vec![uplink.uplink_cidr.into()], - switch: uplink.switch, - port: uplink.uplink_port, - uplink_port_speed: uplink.uplink_port_speed, - uplink_port_fec: uplink.uplink_port_fec, - bgp_peers: vec![], - }], - bgp: vec![], - }), - }, - }; - - assert_eq!(expected, v1); - } - - #[test] - fn serialized_early_network_config_v0_to_v1_conversion_with_vid() { - let v0 = EarlyNetworkConfigV0 { - generation: 1, - rack_subnet: Ipv6Addr::UNSPECIFIED, - ntp_servers: Vec::new(), - rack_network_config: Some(RackNetworkConfigV0 { - infra_ip_first: Ipv4Addr::UNSPECIFIED, - infra_ip_last: Ipv4Addr::UNSPECIFIED, - uplinks: vec![UplinkConfig { - gateway_ip: Ipv4Addr::UNSPECIFIED, - switch: SwitchLocation::Switch0, - uplink_port: "Port0".to_string(), - uplink_port_speed: PortSpeed::Speed100G, - uplink_port_fec: PortFec::None, - uplink_cidr: "192.168.0.1/16".parse().unwrap(), - uplink_vid: Some(10), - }], - }), - }; - - let v0_serialized = serde_json::to_vec(&v0).unwrap(); - let bootstore_conf = - bootstore::NetworkConfig { generation: 1, blob: v0_serialized }; - - let v1 = EarlyNetworkConfig::try_from(bootstore_conf).unwrap(); - let v0_rack_network_config = v0.rack_network_config.unwrap(); - let uplink = v0_rack_network_config.uplinks[0].clone(); - let expected = EarlyNetworkConfig { - generation: 1, - schema_version: 1, - body: EarlyNetworkConfigBody { - ntp_servers: v0.ntp_servers.clone(), - rack_network_config: Some(RackNetworkConfigV1 { - rack_subnet: Ipv6Network::new(v0.rack_subnet, 56).unwrap(), - infra_ip_first: v0_rack_network_config.infra_ip_first, - infra_ip_last: v0_rack_network_config.infra_ip_last, - ports: vec![PortConfigV1 { - routes: vec![RouteConfig { - destination: "0.0.0.0/0".parse().unwrap(), - nexthop: uplink.gateway_ip.into(), - vid: Some(10), }], addresses: vec![uplink.uplink_cidr.into()], switch: uplink.switch, diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index b54a8f0ba0..7f6469d2c0 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -591,7 +591,6 @@ impl ServiceInner { .map(|r| NexusTypes::RouteConfig { destination: r.destination, nexthop: r.nexthop, - vid: r.vid, }) .collect(), addresses: config.addresses.clone(), diff --git a/wicket/src/rack_setup/config_template.toml b/wicket/src/rack_setup/config_template.toml index da07f42cd4..617b61fadc 100644 --- a/wicket/src/rack_setup/config_template.toml +++ b/wicket/src/rack_setup/config_template.toml @@ -47,9 +47,6 @@ infra_ip_last = "" [[rack_network_config.ports]] # Routes associated with this port. # { nexthop = "1.2.3.4", destination = "0.0.0.0/0" } -# Can also optionally specify a VLAN id if the next hop is reachable -# over an 802.1Q tagged L2 segment. -# { nexthop = "5.6.7.8", destination = "5.6.7.0/24", vid = 5 } routes = [] # Addresses associated with this port. diff --git a/wicket/src/rack_setup/config_toml.rs b/wicket/src/rack_setup/config_toml.rs index 9616939308..e087c9aa7c 100644 --- a/wicket/src/rack_setup/config_toml.rs +++ b/wicket/src/rack_setup/config_toml.rs @@ -245,12 +245,6 @@ fn populate_network_table( r.destination.to_string(), )), ); - if let Some(vid) = r.vid { - route.insert( - "vid", - Value::Integer(Formatted::new(vid.into())), - ); - } routes.push(Value::InlineTable(route)); } uplink.insert("routes", Item::Value(Value::Array(routes))); @@ -385,7 +379,6 @@ mod tests { .map(|r| InternalRouteConfig { destination: r.destination, nexthop: r.nexthop, - vid: r.vid, }) .collect(), addresses: config.addresses.clone(), @@ -485,18 +478,10 @@ mod tests { infra_ip_last: "172.30.0.10".parse().unwrap(), ports: vec![PortConfigV1 { addresses: vec!["172.30.0.1/24".parse().unwrap()], - routes: vec![ - RouteConfig { - destination: "0.0.0.0/0".parse().unwrap(), - nexthop: "172.30.0.10".parse().unwrap(), - vid: None, - }, - RouteConfig { - destination: "10.20.0.0/16".parse().unwrap(), - nexthop: "10.0.0.20".parse().unwrap(), - vid: Some(20), - }, - ], + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: "172.30.0.10".parse().unwrap(), + }], bgp_peers: vec![BgpPeerConfig { asn: 47, addr: "10.2.3.4".parse().unwrap(), diff --git a/wicket/src/ui/panes/rack_setup.rs b/wicket/src/ui/panes/rack_setup.rs index 36febe404f..086d01ce9d 100644 --- a/wicket/src/ui/panes/rack_setup.rs +++ b/wicket/src/ui/panes/rack_setup.rs @@ -718,12 +718,7 @@ fn rss_config_text<'a>( vec![ Span::styled(" • Route : ", label_style), Span::styled( - format!( - "{} -> {} (vid={})", - r.destination, - r.nexthop, - r.vid.unwrap_or(0), - ), + format!("{} -> {}", r.destination, r.nexthop), ok_style, ), ] diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index d446ed71d1..a96acc56a0 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -511,7 +511,6 @@ fn validate_rack_network_config( .map(|r| BaRouteConfig { destination: r.destination, nexthop: r.nexthop, - vid: r.vid, }) .collect(), addresses: config.addresses.clone(), From 0f51441b8fc5264ee6ab7f9118722b18e4a1198f Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Mon, 23 Oct 2023 22:30:07 -0700 Subject: [PATCH 80/89] bump version to 1.0.3 in package.sh (#4324) dogfood wants a 1.0.3 Co-authored-by: Alan Hanson --- .github/buildomat/jobs/package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index c1cb04124d..5cfb6fcfd7 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -37,7 +37,7 @@ rustc --version # trampoline global zone images. # COMMIT=$(git rev-parse HEAD) -VERSION="1.0.2-0.ci+git${COMMIT:0:11}" +VERSION="1.0.3-0.ci+git${COMMIT:0:11}" echo "$VERSION" >/work/version.txt ptime -m ./tools/install_builder_prerequisites.sh -yp From 40d0981d60afc389f109e0a6cc6eddb8e5c76aa3 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:21:52 -0700 Subject: [PATCH 81/89] chore(deps): update rust crate semver to 1.0.20 (#4242) --- Cargo.lock | 30 +++++++++++++++--------------- Cargo.toml | 2 +- workspace-hack/Cargo.toml | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7dbaa3e008..84d908140a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -866,7 +866,7 @@ checksum = "fb9ac64500cc83ce4b9f8dafa78186aa008c8dea77a09b94cd307fd0cd5022a8" dependencies = [ "camino", "cargo-platform", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "thiserror", @@ -4388,7 +4388,7 @@ dependencies = [ "rand 0.8.5", "ref-cast", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "sled-agent-client", @@ -4911,7 +4911,7 @@ dependencies = [ "reqwest", "ring", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_derive", "serde_human_bytes", @@ -4953,7 +4953,7 @@ dependencies = [ "reqwest", "ring", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_derive", "serde_human_bytes", @@ -5149,7 +5149,7 @@ dependencies = [ "rustls", "samael", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "serde_urlencoded", @@ -5237,7 +5237,7 @@ dependencies = [ "rayon", "reqwest", "ring", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_derive", "sled-hardware", @@ -5350,7 +5350,7 @@ dependencies = [ "rcgen", "reqwest", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "serial_test", @@ -5483,7 +5483,7 @@ dependencies = [ "ring", "rustix 0.38.9", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "sha2", @@ -5529,7 +5529,7 @@ dependencies = [ "flate2", "futures-util", "reqwest", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_derive", "tar", @@ -7466,7 +7466,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.18", + "semver 1.0.20", ] [[package]] @@ -7750,9 +7750,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" dependencies = [ "serde", ] @@ -9505,8 +9505,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", - "rand 0.4.6", + "cfg-if 1.0.0", + "rand 0.8.5", "static_assertions", ] @@ -10047,7 +10047,7 @@ dependencies = [ "ratatui", "reqwest", "rpassword", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "shell-words", diff --git a/Cargo.toml b/Cargo.toml index 58d57d15b8..d006d9524a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -305,7 +305,7 @@ rustls = "0.21.7" samael = { git = "https://github.com/njaremko/samael", features = ["xmlsec"], branch = "master" } schemars = "0.8.12" secrecy = "0.8.0" -semver = { version = "1.0.18", features = ["std", "serde"] } +semver = { version = "1.0.20", features = ["std", "serde"] } serde = { version = "1.0", default-features = false, features = [ "derive" ] } serde_derive = "1.0" serde_human_bytes = { git = "http://github.com/oxidecomputer/serde_human_bytes", branch = "main" } diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 45c6419d2a..1c3a0b0c73 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -77,7 +77,7 @@ regex-syntax = { version = "0.7.5" } reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } -semver = { version = "1.0.18", features = ["serde"] } +semver = { version = "1.0.20", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.107", features = ["raw_value"] } sha2 = { version = "0.10.7", features = ["oid"] } @@ -170,7 +170,7 @@ regex-syntax = { version = "0.7.5" } reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } -semver = { version = "1.0.18", features = ["serde"] } +semver = { version = "1.0.20", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.107", features = ["raw_value"] } sha2 = { version = "0.10.7", features = ["oid"] } From 672883e3282aceb91166d9497d6301b20e2c1719 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:22:18 -0700 Subject: [PATCH 82/89] chore(deps): update rust crate tokio-util to 0.7.9 (#4268) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84d908140a..395eeb1e79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9119,9 +9119,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index d006d9524a..0b1dee821e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -358,7 +358,7 @@ tokio = "1.33.0" tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] } tokio-stream = "0.1.14" tokio-tungstenite = "0.18" -tokio-util = "0.7.8" +tokio-util = "0.7.9" toml = "0.7.8" toml_edit = "0.19.15" topological-sort = "0.2.2" From b6ec68f38873198cd6e9e8638d306a98f6d882d5 Mon Sep 17 00:00:00 2001 From: David Crespo Date: Tue, 24 Oct 2023 12:22:40 -0500 Subject: [PATCH 83/89] [nexus] [minor] Bring BGP endpoint doc comments in line with conventions (#4329) I noticed there were periods in the docs sidebar. Also used the word "list" as appropriate. --- nexus/src/external_api/http_entrypoints.rs | 16 ++++++++-------- openapi/nexus.json | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 990704904a..48be2de6b0 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -2609,7 +2609,7 @@ async fn networking_loopback_address_delete( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } -/// Get loopback addresses, optionally filtering by id +/// List loopback addresses #[endpoint { method = GET, path = "/v1/system/networking/loopback-address", @@ -2824,7 +2824,7 @@ async fn networking_switch_port_clear_settings( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } -/// Create a new BGP configuration. +/// Create a new BGP configuration #[endpoint { method = POST, path = "/v1/system/networking/bgp", @@ -2845,7 +2845,7 @@ async fn networking_bgp_config_create( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } -/// Get BGP configurations. +/// List BGP configurations #[endpoint { method = GET, path = "/v1/system/networking/bgp", @@ -2900,7 +2900,7 @@ async fn networking_bgp_status( } //TODO pagination? the normal by-name/by-id stuff does not work here -/// Get imported IPv4 BGP routes. +/// Get imported IPv4 BGP routes #[endpoint { method = GET, path = "/v1/system/networking/bgp-routes-ipv4", @@ -2921,7 +2921,7 @@ async fn networking_bgp_imported_routes_ipv4( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } -/// Delete a BGP configuration. +/// Delete a BGP configuration #[endpoint { method = DELETE, path = "/v1/system/networking/bgp", @@ -2942,7 +2942,7 @@ async fn networking_bgp_config_delete( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } -/// Create a new BGP announce set. +/// Create a new BGP announce set #[endpoint { method = POST, path = "/v1/system/networking/bgp-announce", @@ -2964,7 +2964,7 @@ async fn networking_bgp_announce_set_create( } //TODO pagination? the normal by-name/by-id stuff does not work here -/// Get originated routes for a given BGP configuration. +/// Get originated routes for a BGP configuration #[endpoint { method = GET, path = "/v1/system/networking/bgp-announce", @@ -2990,7 +2990,7 @@ async fn networking_bgp_announce_set_list( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } -/// Delete a BGP announce set. +/// Delete a BGP announce set #[endpoint { method = DELETE, path = "/v1/system/networking/bgp-announce", diff --git a/openapi/nexus.json b/openapi/nexus.json index 456f2aebd6..f1bfa4351f 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -5170,7 +5170,7 @@ "tags": [ "system/networking" ], - "summary": "Get BGP configurations.", + "summary": "List BGP configurations", "operationId": "networking_bgp_config_list", "parameters": [ { @@ -5235,7 +5235,7 @@ "tags": [ "system/networking" ], - "summary": "Create a new BGP configuration.", + "summary": "Create a new BGP configuration", "operationId": "networking_bgp_config_create", "requestBody": { "content": { @@ -5270,7 +5270,7 @@ "tags": [ "system/networking" ], - "summary": "Delete a BGP configuration.", + "summary": "Delete a BGP configuration", "operationId": "networking_bgp_config_delete", "parameters": [ { @@ -5301,7 +5301,7 @@ "tags": [ "system/networking" ], - "summary": "Get originated routes for a given BGP configuration.", + "summary": "Get originated routes for a BGP configuration", "operationId": "networking_bgp_announce_set_list", "parameters": [ { @@ -5341,7 +5341,7 @@ "tags": [ "system/networking" ], - "summary": "Create a new BGP announce set.", + "summary": "Create a new BGP announce set", "operationId": "networking_bgp_announce_set_create", "requestBody": { "content": { @@ -5376,7 +5376,7 @@ "tags": [ "system/networking" ], - "summary": "Delete a BGP announce set.", + "summary": "Delete a BGP announce set", "operationId": "networking_bgp_announce_set_delete", "parameters": [ { @@ -5407,7 +5407,7 @@ "tags": [ "system/networking" ], - "summary": "Get imported IPv4 BGP routes.", + "summary": "Get imported IPv4 BGP routes", "operationId": "networking_bgp_imported_routes_ipv4", "parameters": [ { @@ -5482,7 +5482,7 @@ "tags": [ "system/networking" ], - "summary": "Get loopback addresses, optionally filtering by id", + "summary": "List loopback addresses", "operationId": "networking_loopback_address_list", "parameters": [ { From a0fccaff7f36a64d8c270896720fb75027c73ca8 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:22:50 -0700 Subject: [PATCH 84/89] chore(deps): update rust crate flate2 to 1.0.28 (#4280) Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- workspace-hack/Cargo.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 395eeb1e79..4429a5031f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2520,9 +2520,9 @@ checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", diff --git a/Cargo.toml b/Cargo.toml index 0b1dee821e..64f4d5af61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -187,7 +187,7 @@ either = "1.9.0" expectorate = "1.1.0" fatfs = "0.3.6" filetime = "0.2.22" -flate2 = "1.0.27" +flate2 = "1.0.28" flume = "0.11.0" foreign-types = "0.3.2" fs-err = "2.9.0" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 1c3a0b0c73..28aa7a6110 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -35,7 +35,7 @@ crypto-common = { version = "0.1.6", default-features = false, features = ["getr diesel = { version = "2.1.1", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0" } -flate2 = { version = "1.0.27" } +flate2 = { version = "1.0.28" } futures = { version = "0.3.28" } futures-channel = { version = "0.3.28", features = ["sink"] } futures-core = { version = "0.3.28" } @@ -128,7 +128,7 @@ crypto-common = { version = "0.1.6", default-features = false, features = ["getr diesel = { version = "2.1.1", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0" } -flate2 = { version = "1.0.27" } +flate2 = { version = "1.0.28" } futures = { version = "0.3.28" } futures-channel = { version = "0.3.28", features = ["sink"] } futures-core = { version = "0.3.28" } From e14b7379089e6b7b6fafe43306ff8c1ba5b0b45e Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:24:56 -0700 Subject: [PATCH 85/89] chore(deps): update rust crate bcs to 0.1.6 (#4279) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4429a5031f..dd4fa4d69b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,9 +484,9 @@ dependencies = [ [[package]] name = "bcs" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd3ffe8b19a604421a5d461d4a70346223e535903fbc3067138bddbebddcf77" +checksum = "85b6598a2f5d564fb7855dc6b06fd1c38cff5a72bd8b863a4d021938497b440a" dependencies = [ "serde", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 64f4d5af61..62d6712d7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -145,7 +145,7 @@ authz-macros = { path = "nexus/authz-macros" } backoff = { version = "0.4.0", features = [ "tokio" ] } base64 = "0.21.4" bb8 = "0.8.1" -bcs = "0.1.5" +bcs = "0.1.6" bincode = "1.3.3" bootstore = { path = "bootstore" } bootstrap-agent-client = { path = "clients/bootstrap-agent-client" } From c3c968d362f4a8da8ca32220db297727f3f8b7e0 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:29:15 -0700 Subject: [PATCH 86/89] chore(deps): update rust crate openssl-probe to 0.1.5 (#4253) Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com> --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 62d6712d7d..7eb0373d6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -259,7 +259,7 @@ openapiv3 = "1.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -openssl-probe = "0.1.2" +openssl-probe = "0.1.5" opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" } oso = "0.26" owo-colors = "3.5.0" From 35a7f447c8be0b867f2bd923835d87c541b3f809 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:30:23 -0700 Subject: [PATCH 87/89] chore(deps): update rust crate sha2 to 0.10.8 (#4267) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- workspace-hack/Cargo.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd4fa4d69b..191db4715d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8008,9 +8008,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if 1.0.0", "cpufeatures", diff --git a/Cargo.toml b/Cargo.toml index 7eb0373d6c..73f89d3bdf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ serde_tokenstream = "0.2" serde_urlencoded = "0.7.1" serde_with = "2.3.3" serial_test = "0.10" -sha2 = "0.10.7" +sha2 = "0.10.8" sha3 = "0.10.8" shell-words = "1.1.0" signal-hook = "0.3" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 28aa7a6110..884fa14431 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -80,7 +80,7 @@ schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.20", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.107", features = ["raw_value"] } -sha2 = { version = "0.10.7", features = ["oid"] } +sha2 = { version = "0.10.8", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } @@ -173,7 +173,7 @@ schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.20", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.107", features = ["raw_value"] } -sha2 = { version = "0.10.7", features = ["oid"] } +sha2 = { version = "0.10.8", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } From 4bd19c83048b56568208c3b7b159a697653f99fc Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Tue, 24 Oct 2023 10:54:31 -0700 Subject: [PATCH 88/89] [wicketd] Fall back to old behavior if SP is too old to support reading RoT CMPA/CFPA (#4326) We hit this on rack3 but did not hit it on dogfood due to more frequent updates to dogfood's SP/RoT. Prior to this PR, wicketd expected to be able to ask an SP for its RoT's CMPA/CFPA pages, but if a rack is jumping from the 1.0.2 release to current master, its SPs are too old to understand that message. With this change, we will fall back to wicketd's previous behavior of requiring exactly 1 RoT archive if we fail to fetch the CMPA from the target component. --- wicketd/src/update_tracker.rs | 193 +++++++++++++-------- wicketd/tests/integration_tests/updates.rs | 5 +- 2 files changed, 123 insertions(+), 75 deletions(-) diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index e968d65a30..18b692703c 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -1608,7 +1608,13 @@ impl UpdateContext { page.copy_from_slice(&output_buf[..n]); Ok(page) }; - let cmpa = self + + // We may be talking to an SP / RoT that doesn't yet know how to give us + // its CMPA. If we hit a protocol version error here, we can fall back + // to our old behavior of requiring exactly 1 RoT artifact. + let mut artifact_to_apply = None; + + let cmpa = match self .mgs_client .sp_rot_cmpa_get( self.sp.type_, @@ -1616,82 +1622,125 @@ impl UpdateContext { SpComponent::ROT.const_as_str(), ) .await - .map_err(|err| UpdateTerminalError::GetRotCmpaFailed { - error: err.into(), - }) - .and_then(|response| { + { + Ok(response) => { let data = response.into_inner().base64_data; - base64_decode_rot_page(data).map_err(|error| { + Some(base64_decode_rot_page(data).map_err(|error| { UpdateTerminalError::GetRotCmpaFailed { error } - }) - })?; - let cfpa = self - .mgs_client - .sp_rot_cfpa_get( - self.sp.type_, - self.sp.slot, - SpComponent::ROT.const_as_str(), - &gateway_client::types::GetCfpaParams { - slot: RotCfpaSlot::Active, - }, - ) - .await - .map_err(|err| UpdateTerminalError::GetRotCfpaFailed { - error: err.into(), - }) - .and_then(|response| { - let data = response.into_inner().base64_data; - base64_decode_rot_page(data).map_err(|error| { - UpdateTerminalError::GetRotCfpaFailed { error } - }) - })?; - - // Loop through our possible artifacts and find the first (we only - // expect one!) that verifies against the RoT's CMPA/CFPA. - let mut artifact_to_apply = None; - for artifact in available_artifacts { - let image = artifact - .data - .reader_stream() - .and_then(|stream| async { - let mut buf = Vec::with_capacity(artifact.data.file_size()); - StreamReader::new(stream) - .read_to_end(&mut buf) - .await - .context("I/O error reading extracted archive")?; - Ok(buf) - }) - .await - .map_err(|error| { - UpdateTerminalError::FailedFindingSignedRotImage { error } - })?; - let archive = RawHubrisArchive::from_vec(image).map_err(|err| { - UpdateTerminalError::FailedFindingSignedRotImage { - error: anyhow::Error::new(err).context(format!( - "failed to read hubris archive for {:?}", - artifact.id - )), - } - })?; - match archive.verify(&cmpa, &cfpa) { - Ok(()) => { + })?) + } + // TODO is there a better way to check the _specific_ error response + // we get here? We only have a couple of strings; we could check the + // error string contents for something like "WrongVersion", but + // that's pretty fragile. Instead we'll treat any error response + // here as a "fallback to previous behavior". + Err(err @ gateway_client::Error::ErrorResponse(_)) => { + if available_artifacts.len() == 1 { info!( - self.log, "RoT archive verification success"; - "name" => artifact.id.name.as_str(), - "version" => %artifact.id.version, - "kind" => ?artifact.id.kind, + self.log, + "Failed to get RoT CMPA page; \ + using only available RoT artifact"; + "err" => %err, ); - artifact_to_apply = Some(artifact.clone()); - break; - } - Err(err) => { - // We log this but don't fail - we want to continue looking - // for a verifiable artifact. - info!( - self.log, "RoT archive verification failed"; - "artifact" => ?artifact.id, - "err" => %DisplayErrorChain::new(&err), + artifact_to_apply = Some(available_artifacts[0].clone()); + None + } else { + error!( + self.log, + "Failed to get RoT CMPA; unable to choose from \ + multiple available RoT artifacts"; + "err" => %err, + "num_rot_artifacts" => available_artifacts.len(), ); + return Err(UpdateTerminalError::GetRotCmpaFailed { + error: err.into(), + }); + } + } + // For any other error (e.g., comms failures), just fail as normal. + Err(err) => { + return Err(UpdateTerminalError::GetRotCmpaFailed { + error: err.into(), + }); + } + }; + + // If we were able to fetch the CMPA, we also need to fetch the CFPA and + // then find the correct RoT artifact. If we weren't able to fetch the + // CMPA, we either already set `artifact_to_apply` to the one-and-only + // RoT artifact available, or we returned a terminal error. + if let Some(cmpa) = cmpa { + let cfpa = self + .mgs_client + .sp_rot_cfpa_get( + self.sp.type_, + self.sp.slot, + SpComponent::ROT.const_as_str(), + &gateway_client::types::GetCfpaParams { + slot: RotCfpaSlot::Active, + }, + ) + .await + .map_err(|err| UpdateTerminalError::GetRotCfpaFailed { + error: err.into(), + }) + .and_then(|response| { + let data = response.into_inner().base64_data; + base64_decode_rot_page(data).map_err(|error| { + UpdateTerminalError::GetRotCfpaFailed { error } + }) + })?; + + // Loop through our possible artifacts and find the first (we only + // expect one!) that verifies against the RoT's CMPA/CFPA. + for artifact in available_artifacts { + let image = artifact + .data + .reader_stream() + .and_then(|stream| async { + let mut buf = + Vec::with_capacity(artifact.data.file_size()); + StreamReader::new(stream) + .read_to_end(&mut buf) + .await + .context("I/O error reading extracted archive")?; + Ok(buf) + }) + .await + .map_err(|error| { + UpdateTerminalError::FailedFindingSignedRotImage { + error, + } + })?; + let archive = + RawHubrisArchive::from_vec(image).map_err(|err| { + UpdateTerminalError::FailedFindingSignedRotImage { + error: anyhow::Error::new(err).context(format!( + "failed to read hubris archive for {:?}", + artifact.id + )), + } + })?; + match archive.verify(&cmpa, &cfpa) { + Ok(()) => { + info!( + self.log, "RoT archive verification success"; + "name" => artifact.id.name.as_str(), + "version" => %artifact.id.version, + "kind" => ?artifact.id.kind, + ); + artifact_to_apply = Some(artifact.clone()); + break; + } + Err(err) => { + // We log this but don't fail - we want to continue + // looking for a verifiable artifact. + info!( + self.log, "RoT archive verification failed"; + "artifact" => ?artifact.id, + "err" => %DisplayErrorChain::new(&err), + ); + } } } } diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs index af743190c2..a9be9d4747 100644 --- a/wicketd/tests/integration_tests/updates.rs +++ b/wicketd/tests/integration_tests/updates.rs @@ -168,9 +168,8 @@ async fn test_updates() { match terminal_event.kind { StepEventKind::ExecutionFailed { failed_step, .. } => { // TODO: obviously we shouldn't stop here, get past more of the - // update process in this test. We currently fail when attempting to - // look up the RoT's CMPA/CFPA. - assert_eq!(failed_step.info.component, UpdateComponent::Rot); + // update process in this test. + assert_eq!(failed_step.info.component, UpdateComponent::Sp); } other => { panic!("unexpected terminal event kind: {other:?}"); From ca1b0ba216ed8af485e46a014399e3adb00abb90 Mon Sep 17 00:00:00 2001 From: Rain Date: Tue, 24 Oct 2023 12:01:30 -0700 Subject: [PATCH 89/89] [renovate] use actions/pin checked into our global renovate-config (#4320) See https://github.com/oxidecomputer/renovate-config/blob/main/actions/pin.json. --- .github/renovate.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/renovate.json b/.github/renovate.json index 405a3e282b..606be44c87 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -4,6 +4,6 @@ "local>oxidecomputer/renovate-config", "local>oxidecomputer/renovate-config//rust/autocreate", "local>oxidecomputer/renovate-config:post-upgrade", - "helpers:pinGitHubActionDigests" + "local>oxidecomputer/renovate-config//actions/pin" ] }