Rebase

Created using spr 1.3.4

.config/nextest.toml

@@ -3,7 +3,16 @@
 #
 # The required version should be bumped up if we need new features, performance
 # improvements or bugfixes that are present in newer versions of nextest.
-nextest-version = { required = "0.9.55", recommended = "0.9.57" }
+nextest-version = { required = "0.9.59", recommended = "0.9.59" }
+
+experimental = ["setup-scripts"]
+
+[[profile.default.scripts]]
+filter = 'rdeps(nexus-test-utils)'
+setup = 'crdb-seed'
 
 [profile.ci]
 fail-fast = false
+
+[script.crdb-seed]
+command = 'cargo run -p crdb-seed'

.github/buildomat/build-and-test.sh

@@ -7,7 +7,7 @@ set -o xtrace
 # NOTE: This version should be in sync with the recommended version in
 # .config/nextest.toml. (Maybe build an automated way to pull the recommended
 # version in the future.)
-NEXTEST_VERSION='0.9.57'
+NEXTEST_VERSION='0.9.59'
 
 cargo --version
 rustc --version
@@ -66,6 +66,11 @@ ptime -m timeout 2h cargo nextest run --profile ci --locked --verbose
 banner doctest
 ptime -m timeout 1h cargo test --doc --locked --verbose --no-fail-fast
 
+# We expect the seed CRDB to be placed here, so we explicitly remove it so the
+# rmdir check below doesn't get triggered. Nextest doesn't have support for
+# teardown scripts so this is the best we've got.
+rm -rf "$TEST_TMPDIR/crdb-base"
+
 #
 # Make sure that we have left nothing around in $TEST_TMPDIR.  The easiest way
 # to check is to try to remove it with `rmdir`.

Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "caboose-util",
     "certificates",
     "common",
+    "crdb-seed",
     "ddm-admin-client",
     "deploy",
     "dev-tools/omdb",

crdb-seed/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "crdb-seed"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]
+camino.workspace = true
+camino-tempfile.workspace = true
+dropshot.workspace = true
+hex.workspace = true
+omicron-test-utils.workspace = true
+ring.workspace = true
+slog.workspace = true
+tokio.workspace = true

crdb-seed/src/main.rs

@@ -0,0 +1,92 @@
+use camino::Utf8PathBuf;
+use dropshot::{test_util::LogContext, ConfigLogging, ConfigLoggingLevel};
+use omicron_test_utils::dev;
+use slog::Logger;
+use std::io::Write;
+
+// Creates a string identifier for the current DB schema and version.
+//
+// The goal here is to allow to create different "seed" directories
+// for each revision of the DB.
+fn digest_unique_to_schema() -> String {
+    let schema = include_str!("../../schema/crdb/dbinit.sql");
+    let crdb_version = include_str!("../../tools/cockroachdb_version");
+    let mut ctx = ring::digest::Context::new(&ring::digest::SHA256);
+    ctx.update(&schema.as_bytes());
+    ctx.update(&crdb_version.as_bytes());
+    let digest = ctx.finish();
+    hex::encode(digest.as_ref())
+}
+
+enum SeedDirectoryStatus {
+    Created,
+    Existing,
+}
+
+async fn ensure_seed_directory_exists(
+    log: &Logger,
+) -> (Utf8PathBuf, SeedDirectoryStatus) {
+    let base_seed_dir = Utf8PathBuf::from_path_buf(std::env::temp_dir())
+        .expect("Not a UTF-8 path")
+        .join("crdb-base");
+    std::fs::create_dir_all(&base_seed_dir).unwrap();
+    let desired_seed_dir = base_seed_dir.join(digest_unique_to_schema());
+
+    if desired_seed_dir.exists() {
+        return (desired_seed_dir, SeedDirectoryStatus::Existing);
+    }
+
+    // The directory didn't exist when we started, so try to create it.
+    //
+    // Nextest will execute it just once, but it is possible for a user to start
+    // up multiple nextest processes to be running at the same time. So we
+    // should consider it possible for another caller to create this seed
+    // directory before we finish setting it up ourselves.
+    let tmp_seed_dir =
+        camino_tempfile::Utf8TempDir::new_in(base_seed_dir).unwrap();
+    dev::test_setup_database_seed(log, tmp_seed_dir.path()).await;
+
+    // If we can successfully perform the rename, there was either no
+    // contention or we won a creation race.
+    //
+    // If we couldn't perform the rename, the directory might already exist.
+    // Check that this is the error we encountered -- otherwise, we're
+    // struggling.
+    if let Err(err) = std::fs::rename(tmp_seed_dir.path(), &desired_seed_dir) {
+        if !desired_seed_dir.exists() {
+            panic!("Cannot rename seed directory for CockroachDB: {err}");
+        }
+    }
+
+    (desired_seed_dir, SeedDirectoryStatus::Created)
+}
+
+#[tokio::main]
+async fn main() {
+    // TODO: dropshot is v heavyweight for this, we should be able to pull in a
+    // smaller binary
+    let logctx = LogContext::new(
+        "crdb_seeding",
+        &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info },
+    );
+    let (dir, status) = ensure_seed_directory_exists(&logctx.log).await;
+    match status {
+        SeedDirectoryStatus::Created => {
+            slog::info!(logctx.log, "Created seed directory: `{dir}`");
+        }
+        SeedDirectoryStatus::Existing => {
+            slog::info!(logctx.log, "Using existing seed directory: `{dir}`");
+        }
+    }
+    if let Ok(env_path) = std::env::var("NEXTEST_ENV") {
+        let mut file = std::fs::File::create(&env_path)
+            .expect("failed to open NEXTEST_ENV file");
+        writeln!(file, "CRDB_SEED_DIR={dir}")
+            .expect("failed to write to NEXTEST_ENV file");
+    } else {
+        slog::warn!(
+            logctx.log,
+            "NEXTEST_ENV not set (is this script running under nextest?)"
+        );
+    }
+}

nexus/test-utils/Cargo.toml

@@ -39,8 +39,3 @@ trust-dns-proto.workspace = true
 trust-dns-resolver.workspace = true
 uuid.workspace = true
 omicron-workspace-hack = { version = "0.1", path = "../../workspace-hack" }
-
-[build-dependencies]
-dropshot.workspace = true
-omicron-test-utils.workspace = true
-tokio.workspace = true

nexus/test-utils/src/db.rs

@@ -4,9 +4,9 @@
 
 //! Database testing facilities.
 
+use camino::Utf8PathBuf;
 use omicron_test_utils::dev;
 use slog::Logger;
-use std::path::PathBuf;
 
 /// Path to the "seed" CockroachDB directory.
 ///
@@ -16,8 +16,11 @@ use std::path::PathBuf;
 /// By creating a "seed" version of the database, we can cut down
 /// on the time spent performing this operation. Instead, we opt
 /// to copy the database from this seed location.
-fn seed_dir() -> PathBuf {
-    PathBuf::from(concat!(env!("OUT_DIR"), "/crdb-base"))
+fn seed_dir() -> Utf8PathBuf {
+    // The setup script should set this environment variable.
+    let seed_dir = std::env::var("CRDB_SEED_DIR")
+        .expect("CRDB_SEED_DIR missing -- are you running this test with `cargo nextest run`?");
+    seed_dir.into()
 }
 
 /// Wrapper around [`dev::test_setup_database`] which uses a a

test-utils/Cargo.toml

@@ -6,6 +6,7 @@ license = "MPL-2.0"
 
 [dependencies]
 anyhow.workspace = true
+camino.workspace = true
 dropshot.workspace = true
 futures.workspace = true
 headers.workspace = true

test-utils/src/dev/mod.rs

@@ -12,12 +12,14 @@ pub mod poll;
 pub mod test_cmds;
 
 use anyhow::Context;
+use camino::Utf8Path;
+use camino::Utf8PathBuf;
 pub use dropshot::test_util::LogContext;
 use dropshot::ConfigLogging;
 use dropshot::ConfigLoggingIfExists;
 use dropshot::ConfigLoggingLevel;
 use slog::Logger;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 
 // Helper for copying all the files in one directory to another.
 fn copy_dir(
@@ -77,22 +79,22 @@ pub enum StorageSource {
     /// Do not populate anything. This is primarily used for migration testing.
     DoNotPopulate,
     /// Populate the latest version of the database.
-    PopulateLatest { output_dir: PathBuf },
+    PopulateLatest { output_dir: Utf8PathBuf },
     /// Copy the database from a seed directory, which has previously
     /// been created with `PopulateLatest`.
-    CopyFromSeed { input_dir: PathBuf },
+    CopyFromSeed { input_dir: Utf8PathBuf },
 }
 
 /// Creates a [`db::CockroachInstance`] with a populated storage directory.
 ///
 /// This is intended to optimize subsequent calls to [`test_setup_database`]
 /// by reducing the latency of populating the storage directory.
-pub async fn test_setup_database_seed(log: &Logger, dir: &Path) {
-    let _ = std::fs::remove_dir_all(&dir);
-    std::fs::create_dir_all(&dir).unwrap();
+pub async fn test_setup_database_seed(log: &Logger, dir: &Utf8Path) {
+    let _ = std::fs::remove_dir_all(dir);
+    std::fs::create_dir_all(dir).unwrap();
     let mut db = setup_database(
         log,
-        StorageSource::PopulateLatest { output_dir: dir.to_path_buf() },
+        StorageSource::PopulateLatest { output_dir: dir.to_owned() },
     )
     .await;
     db.cleanup().await.unwrap();
@@ -148,7 +150,7 @@ async fn setup_database(
         StorageSource::CopyFromSeed { input_dir } => {
             info!(&log,
                 "cockroach: copying from seed directory ({}) to storage directory ({})",
-                input_dir.to_string_lossy(), starter.store_dir().to_string_lossy(),
+                input_dir, starter.store_dir().to_string_lossy(),
             );
             copy_dir(input_dir, starter.store_dir())
                 .expect("Cannot copy storage from seed directory");