Scale out Clickhouse to a multinode cluster

.github/buildomat/jobs/package.sh

@@ -104,6 +104,7 @@ ptime -m ./tools/build-global-zone-packages.sh "$tarball_src_dir" /work
 mkdir -p /work/zones
 zones=(
   out/clickhouse.tar.gz
+  out/clickhouse_keeper.tar.gz
   out/cockroachdb.tar.gz
   out/crucible-pantry.tar.gz
   out/crucible.tar.gz

common/src/address.rs

@@ -36,6 +36,7 @@ pub const PROPOLIS_PORT: u16 = 12400;
 pub const COCKROACH_PORT: u16 = 32221;
 pub const CRUCIBLE_PORT: u16 = 32345;
 pub const CLICKHOUSE_PORT: u16 = 8123;
+pub const CLICKHOUSE_KEEPER_PORT: u16 = 9181;
 pub const OXIMETER_PORT: u16 = 12223;
 pub const DENDRITE_PORT: u16 = 12224;
 pub const DDMD_PORT: u16 = 8000;

dev-tools/src/bin/omicron-dev.rs

@@ -268,7 +268,7 @@ async fn cmd_clickhouse_run(args: &ChRunArgs) -> Result<(), anyhow::Error> {
 
     // Start the database server process, possibly on a specific port
     let mut db_instance =
-        dev::clickhouse::ClickHouseInstance::new(args.port).await?;
+        dev::clickhouse::ClickHouseInstance::new_single_node(args.port).await?;
     println!(
         "omicron-dev: running ClickHouse with full command:\n\"clickhouse {}\"",
         db_instance.cmdline().join(" ")

internal-dns-cli/src/bin/dnswait.rs

@@ -23,21 +23,31 @@ struct Opt {
     #[clap(long, action)]
     nameserver_addresses: Vec<SocketAddr>,
 
-    /// service name to be resolved (should be the target of a DNS name)
+    /// Service name to be resolved (should be the target of a DNS name)
     #[arg(value_enum)]
     srv_name: ServiceName,
+
+    /// Output service host names only, omitting the port
+    #[clap(long, short = 'H', action)]
+    hostname_only: bool,
 }
 
 #[derive(Debug, Clone, Copy, ValueEnum)]
 #[value(rename_all = "kebab-case")]
 enum ServiceName {
     Cockroach,
+    Clickhouse,
+    ClickhouseKeeper,
 }
 
 impl From<ServiceName> for internal_dns::ServiceName {
     fn from(value: ServiceName) -> Self {
         match value {
             ServiceName::Cockroach => internal_dns::ServiceName::Cockroach,
+            ServiceName::Clickhouse => internal_dns::ServiceName::Clickhouse,
+            ServiceName::ClickhouseKeeper => {
+                internal_dns::ServiceName::ClickhouseKeeper
+            }
         }
     }
 }
@@ -91,7 +101,11 @@ async fn main() -> Result<()> {
     .context("unexpectedly gave up")?;
 
     for (target, port) in result {
-        println!("{}:{}", target, port)
+        if opt.hostname_only {
+            println!("{}", target)
+        } else {
+            println!("{}:{}", target, port)
+        }
     }
 
     Ok(())

internal-dns/src/config.rs

@@ -422,6 +422,10 @@ mod test {
     #[test]
     fn display_srv_service() {
         assert_eq!(ServiceName::Clickhouse.dns_name(), "_clickhouse._tcp",);
+        assert_eq!(
+            ServiceName::ClickhouseKeeper.dns_name(),
+            "_clickhouse-keeper._tcp",
+        );
         assert_eq!(ServiceName::Cockroach.dns_name(), "_cockroach._tcp",);
         assert_eq!(ServiceName::InternalDns.dns_name(), "_nameservice._tcp",);
         assert_eq!(ServiceName::Nexus.dns_name(), "_nexus._tcp",);

internal-dns/src/names.rs

@@ -17,6 +17,7 @@ pub const DNS_ZONE_EXTERNAL_TESTING: &str = "oxide-dev.test";
 #[derive(Clone, Debug, Hash, Eq, Ord, PartialEq, PartialOrd)]
 pub enum ServiceName {
     Clickhouse,
+    ClickhouseKeeper,
     Cockroach,
     InternalDns,
     ExternalDns,
@@ -38,6 +39,7 @@ impl ServiceName {
     fn service_kind(&self) -> &'static str {
         match self {
             ServiceName::Clickhouse => "clickhouse",
+            ServiceName::ClickhouseKeeper => "clickhouse-keeper",
             ServiceName::Cockroach => "cockroach",
             ServiceName::ExternalDns => "external-dns",
             ServiceName::InternalDns => "nameservice",
@@ -61,6 +63,7 @@ impl ServiceName {
     pub(crate) fn dns_name(&self) -> String {
         match self {
             ServiceName::Clickhouse
+            | ServiceName::ClickhouseKeeper
             | ServiceName::Cockroach
             | ServiceName::InternalDns
             | ServiceName::ExternalDns

nexus/benches/setup_benchmark.rs

@@ -29,7 +29,7 @@ async fn do_crdb_setup() {
 // Wraps exclusively the ClickhouseDB portion of setup/teardown.
 async fn do_clickhouse_setup() {
     let mut clickhouse =
-        dev::clickhouse::ClickHouseInstance::new(0).await.unwrap();
+        dev::clickhouse::ClickHouseInstance::new_single_node(0).await.unwrap();
     clickhouse.cleanup().await.unwrap();
 }
 

nexus/db-model/src/dataset_kind.rs

@@ -19,6 +19,7 @@ impl_enum_type!(
     Crucible => b"crucible"
     Cockroach => b"cockroach"
     Clickhouse => b"clickhouse"
+    ClickhouseKeeper => b"clickhouse_keeper"
     ExternalDns => b"external_dns"
     InternalDns => b"internal_dns"
 );
@@ -35,6 +36,9 @@ impl From<internal_api::params::DatasetKind> for DatasetKind {
             internal_api::params::DatasetKind::Clickhouse => {
                 DatasetKind::Clickhouse
             }
+            internal_api::params::DatasetKind::ClickhouseKeeper => {
+                DatasetKind::ClickhouseKeeper
+            }
             internal_api::params::DatasetKind::ExternalDns => {
                 DatasetKind::ExternalDns
             }

nexus/db-model/src/schema.rs

@@ -1130,7 +1130,7 @@ table! {
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(3, 0, 3);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(4, 0, 0);
 
 allow_tables_to_appear_in_same_query!(
     system_update,

nexus/db-model/src/service_kind.rs

@@ -18,6 +18,7 @@ impl_enum_type!(
 
     // Enum values
     Clickhouse => b"clickhouse"
+    ClickhouseKeeper => b"clickhouse_keeper"
     Cockroach => b"cockroach"
     Crucible => b"crucible"
     CruciblePantry => b"crucible_pantry"
@@ -54,6 +55,9 @@ impl From<internal_api::params::ServiceKind> for ServiceKind {
             internal_api::params::ServiceKind::Clickhouse => {
                 ServiceKind::Clickhouse
             }
+            internal_api::params::ServiceKind::ClickhouseKeeper => {
+                ServiceKind::ClickhouseKeeper
+            }
             internal_api::params::ServiceKind::Cockroach => {
                 ServiceKind::Cockroach
             }

nexus/test-utils/src/lib.rs

@@ -321,7 +321,9 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         let log = &self.logctx.log;
         debug!(log, "Starting Clickhouse");
         let clickhouse =
-            dev::clickhouse::ClickHouseInstance::new(0).await.unwrap();
+            dev::clickhouse::ClickHouseInstance::new_single_node(0)
+                .await
+                .unwrap();
         let port = clickhouse.port();
 
         let zpool_id = Uuid::new_v4();

nexus/tests/integration_tests/oximeter.rs

@@ -110,7 +110,10 @@ async fn test_oximeter_reregistration() {
     );
     let client =
         oximeter_db::Client::new(ch_address.into(), &context.logctx.log);
-    client.init_db().await.expect("Failed to initialize timeseries database");
+    client
+        .init_single_node_db()
+        .await
+        .expect("Failed to initialize timeseries database");
 
     // Helper to retrieve the timeseries from ClickHouse
     let timeseries_name = "integration_target:integration_metric";

nexus/types/src/internal_api/params.rs

@@ -125,6 +125,7 @@ pub enum DatasetKind {
     Crucible,
     Cockroach,
     Clickhouse,
+    ClickhouseKeeper,
     ExternalDns,
     InternalDns,
 }
@@ -136,6 +137,7 @@ impl fmt::Display for DatasetKind {
             Crucible => "crucible",
             Cockroach => "cockroach",
             Clickhouse => "clickhouse",
+            ClickhouseKeeper => "clickhouse_keeper",
             ExternalDns => "external_dns",
             InternalDns => "internal_dns",
         };
@@ -168,6 +170,7 @@ pub struct ServiceNic {
 #[serde(rename_all = "snake_case", tag = "type", content = "content")]
 pub enum ServiceKind {
     Clickhouse,
+    ClickhouseKeeper,
     Cockroach,
     Crucible,
     CruciblePantry,
@@ -186,6 +189,7 @@ impl fmt::Display for ServiceKind {
         use ServiceKind::*;
         let s = match self {
             Clickhouse => "clickhouse",
+            ClickhouseKeeper => "clickhouse_keeper",
             Cockroach => "cockroach",
             Crucible => "crucible",
             ExternalDns { .. } => "external_dns",

openapi/nexus-internal.json

@@ -922,6 +922,7 @@
           "crucible",
           "cockroach",
           "clickhouse",
+          "clickhouse_keeper",
           "external_dns",
           "internal_dns"
         ]
@@ -2803,6 +2804,20 @@
               "type"
             ]
           },
+          {
+            "type": "object",
+            "properties": {
+              "type": {
+                "type": "string",
+                "enum": [
+                  "clickhouse_keeper"
+                ]
+              }
+            },
+            "required": [
+              "type"
+            ]
+          },
           {
             "type": "object",
             "properties": {

openapi/sled-agent.json

@@ -1091,6 +1091,20 @@
               "type"
             ]
           },
+          {
+            "type": "object",
+            "properties": {
+              "type": {
+                "type": "string",
+                "enum": [
+                  "clickhouse_keeper"
+                ]
+              }
+            },
+            "required": [
+              "type"
+            ]
+          },
           {
             "type": "object",
             "properties": {
@@ -2524,6 +2538,24 @@
               "type"
             ]
           },
+          {
+            "type": "object",
+            "properties": {
+              "address": {
+                "type": "string"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "clickhouse_keeper"
+                ]
+              }
+            },
+            "required": [
+              "address",
+              "type"
+            ]
+          },
           {
             "type": "object",
             "properties": {
@@ -3115,6 +3147,7 @@
         "type": "string",
         "enum": [
           "clickhouse",
+          "clickhouse_keeper",
           "cockroach_db",
           "crucible_pantry",
           "crucible",

oximeter/collector/src/lib.rs

@@ -321,7 +321,12 @@ impl OximeterAgent {
             )
         };
         let client = Client::new(db_address, &log);
-        client.init_db().await?;
+        let replicated = client.is_oximeter_cluster().await?;
+        if !replicated {
+            client.init_single_node_db().await?;
+        } else {
+            client.init_replicated_db().await?;
+        }
 
         // Spawn the task for aggregating and inserting all metrics
         tokio::spawn(async move {

oximeter/db/src/bin/oxdb.rs

@@ -148,7 +148,7 @@ async fn make_client(
     let address = SocketAddr::new(address, port);
     let client = Client::new(address, &log);
     client
-        .init_db()
+        .init_single_node_db()
         .await
         .context("Failed to initialize timeseries database")?;
     Ok(client)
@@ -261,13 +261,13 @@ async fn populate(
     Ok(())
 }
 
-async fn wipe_db(
+async fn wipe_single_node_db(
     address: IpAddr,
     port: u16,
     log: Logger,
 ) -> Result<(), anyhow::Error> {
     let client = make_client(address, port, &log).await?;
-    client.wipe_db().await.context("Failed to wipe database")
+    client.wipe_single_node_db().await.context("Failed to wipe database")
 }
 
 async fn query(
@@ -313,7 +313,7 @@ async fn main() {
                 .unwrap();
         }
         Subcommand::Wipe => {
-            wipe_db(args.address, args.port, log).await.unwrap()
+            wipe_single_node_db(args.address, args.port, log).await.unwrap()
         }
         Subcommand::Query {
             timeseries_name,