Optimize CockroachDB setup for tests (96% reduction) (#493)

Optimizes setup time for CRDB-based tests by seeding + copying a seeded database, rather than populating the database anew during each test. This significantly cuts down on the setup time for any test which wants to use even a simple, single-node Cockroach instance.

.github/buildomat/jobs/build-and-test.sh

@@ -14,6 +14,18 @@ set -o xtrace
 cargo --version
 rustc --version
 
+banner clickhouse
+ptime -m ./tools/ci_download_clickhouse
+
+banner cockroach
+ptime -m bash ./tools/ci_download_cockroachdb
+
+#
+# Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
+# suite.
+#
+export PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse"
+
 #
 # We build with:
 #
@@ -39,18 +51,6 @@ ptime -m cargo +'nightly-2021-11-24' build --locked --all-targets --verbose
 banner deploy-check
 ptime -m cargo run --bin omicron-package -- check
 
-banner clickhouse
-ptime -m ./tools/ci_download_clickhouse
-
-banner cockroach
-ptime -m bash ./tools/ci_download_cockroachdb
-
-#
-# Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test
-# suite.
-#
-export PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse"
-
 #
 # NOTE: We're using using the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid
 # having to rebuild here.

.github/workflows/rust.yml

@@ -108,6 +108,12 @@ jobs:
       with:
         key: ${{ runner.os }}-clickhouse-binary-${{ hashFiles('tools/clickhouse_checksums') }}
         path: "clickhouse"
+    - name: Download ClickHouse
+      if: steps.cache-clickhouse.outputs.cache-hit != 'true'
+      run: ./tools/ci_download_clickhouse
+    - name: Download CockroachDB binary
+      if: steps.cache-cockroachdb.outputs.cache-hit != 'true'
+      run: bash ./tools/ci_download_cockroachdb
     - name: Build
       # We build with:
       # - RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings": disallow warnings
@@ -119,13 +125,7 @@ jobs:
       #   also gives us a record of which dependencies were used for each CI
       #   run.  Building with `--locked` ensures that the checked-in Cargo.lock
       #   is up to date.
-      run: RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo build --locked --all-targets --verbose
-    - name: Download ClickHouse
-      if: steps.cache-clickhouse.outputs.cache-hit != 'true'
-      run: ./tools/ci_download_clickhouse
-    - name: Download CockroachDB binary
-      if: steps.cache-cockroachdb.outputs.cache-hit != 'true'
-      run: bash ./tools/ci_download_cockroachdb
+      run: PATH="$PATH:$PWD/cockroachdb/bin:$PWD/clickhouse" RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo build --locked --all-targets --verbose
     - name: Run tests
       # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid having to
       # rebuild here.

Cargo.toml

@@ -23,7 +23,6 @@ default-members = [
     "common",
     "nexus",
     "nexus/src/db/db-macros",
-    "nexus/test-utils",
     "package",
     "rpaths",
     "sled-agent",

README.adoc

@@ -83,7 +83,7 @@ example, on Helios, you'd want `/usr/bin` on your PATH.
 --
 . CockroachDB v21.1.10.
 +
-The test suite expects to be able to start a single-node CockroachDB cluster using the `cockroach` executable on your PATH.
+The build and test suite expects to be able to start a single-node CockroachDB cluster using the `cockroach` executable on your PATH.
 On illumos, MacOS, and Linux, you should be able to use the `tools/ci_download_cockroachdb` script to fetch the official CockroachDB binary.  It will be put into `./cockroachdb/bin/cockroach`.
 Alternatively, you can follow the https://www.cockroachlabs.com/docs/stable/install-cockroachdb.html[official CockroachDB installation instructions for your platform].
 

nexus/test-utils/Cargo.toml

@@ -8,7 +8,7 @@ license = "MPL-2.0"
 anyhow = "1.0"
 bytes = "1.0.1"
 chrono = { version = "0.4",  features = [ "serde" ] }
-dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main" }
+dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] }
 http = "0.2.5"
 hyper = "0.14"
 omicron-common = { path = "../../common" }
@@ -24,3 +24,8 @@ serde = { version = "1.0",  features = [ "derive" ] }
 serde_json = "1.0"
 slog = { version = "2.7",  features = [ "max_level_trace", "release_max_level_debug" ] }
 uuid = { version = "0.8", features = [ "serde", "v4" ] }
+
+[build-dependencies]
+dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] }
+omicron-test-utils = { path = "../../test-utils" }
+tokio = { version = "1.14" }

nexus/test-utils/build.rs

@@ -0,0 +1,31 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use dropshot::{test_util::LogContext, ConfigLogging, ConfigLoggingLevel};
+use omicron_test_utils::dev::test_setup_database_seed;
+
+// Creates a "pre-populated" CockroachDB storage directory, which
+// subsequent tests can copy instead of creating themselves.
+//
+// Is it critical this happens at build-time? No. However, it
+// makes it more convenient for tests to assume this seeded
+// directory exists, rather than all attempting to create it
+// concurrently.
+//
+// Refer to the documentation of [`test_setup_database_seed`] for
+// more context.
+#[tokio::main]
+async fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=../../common/src/sql/dbinit.sql");
+    println!("cargo:rerun-if-changed=../../tools/cockroachdb_checksums");
+    println!("cargo:rerun-if-changed=../../tools/cockroachdb_version");
+
+    let logctx = LogContext::new(
+        "crdb_seeding",
+        &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info },
+    );
+
+    test_setup_database_seed(&logctx.log).await;
+}