Scale out Clickhouse to a multinode cluster

oximeter/db/src/client.rs

@@ -761,7 +761,7 @@ mod tests {
             .expect("Failed to initialize timeseries database");
 
         // Wait to make sure data has been synchronised.
-        // TODO: Waiting for 5 secs is a bit sloppy,
+        // TODO(https://github.com/oxidecomputer/omicron/issues/4001): Waiting for 5 secs is a bit sloppy,
         // come up with a better way to do this.
         sleep(Duration::from_secs(5)).await;
 

oximeter/db/src/configs/replica_config.xml

@@ -315,7 +315,7 @@
 
     <remote_servers replace="true">
         <oximeter_cluster>
-            <!-- TODO: secret handling TBD -->
+            <!-- TODO(https://github.com/oxidecomputer/omicron/issues/3823): secret handling TBD -->
             <secret>mysecretphrase</secret>
             <shard>
                 <internal_replication>true</internal_replication>

schema/crdb/4.0.0/up.sql

@@ -20,6 +20,5 @@ SELECT CAST(
 );
 
 ALTER TYPE omicron.public.service_kind ADD VALUE IF NOT EXISTS 'clickhouse_keeper';
-ALTER TYPE omicron.public.dataset_kind ADD VALUE IF NOT EXISTS 'clickhouse_keeper';
 
 COMMIT;

schema/crdb/4.0.1/up.sql

@@ -0,0 +1,24 @@
+-- CRDB documentation recommends the following:
+--  "Execute schema changes either as single statements (as an implicit transaction),
+--   or in an explicit transaction consisting of the single schema change statement."
+--
+-- For each schema change, we transactionally:
+-- 1. Check the current version
+-- 2. Apply the idempotent update
+
+BEGIN;
+
+SELECT CAST(
+    IF(
+        (
+            SELECT version = '4.0.0' and target_version = '4.0.1'
+            FROM omicron.public.db_metadata WHERE singleton = true
+        ),
+        'true',
+        'Invalid starting version for schema change'
+    ) AS BOOL
+);
+
+ALTER TYPE omicron.public.dataset_kind ADD VALUE IF NOT EXISTS 'clickhouse_keeper';
+
+COMMIT;

schema/crdb/dbinit.sql

@@ -2562,7 +2562,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '4.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '4.0.1', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;

sled-agent/src/rack_setup/plan/service.rs

@@ -56,11 +56,11 @@ const CRDB_COUNT: usize = 5;
 const OXIMETER_COUNT: usize = 1;
 // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove
 // when Nexus provisions Clickhouse.
-// TODO: Set to 2 once we enable replicated ClickHouse
+// TODO(https://github.com/oxidecomputer/omicron/issues/4000): Set to 2 once we enable replicated ClickHouse
 const CLICKHOUSE_COUNT: usize = 1;
 // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove
 // when Nexus provisions Clickhouse keeper.
-// TODO: Set to 3 once we enable replicated ClickHouse
+// TODO(https://github.com/oxidecomputer/omicron/issues/4000): Set to 3 once we enable replicated ClickHouse
 const CLICKHOUSE_KEEPER_COUNT: usize = 0;
 // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove.
 // when Nexus provisions Crucible.

smf/clickhouse/config_replica.xml

@@ -37,7 +37,7 @@
 
     <remote_servers replace="true">
         <oximeter_cluster>
-            <!-- TODO: secret handling TBD -->
+            <!-- TODO(https://github.com/oxidecomputer/omicron/issues/3823): secret handling TBD -->
             <secret>mysecretphrase</secret>
             <shard>
                 <internal_replication>true</internal_replication>

smf/clickhouse/method_script.sh

@@ -34,7 +34,7 @@ route get -inet6 default -inet6 "$GATEWAY" || route add -inet6 default -inet6 "$
 single_node=true
 
 command=()
-# TODO: Remove single node mode once all racks are running in replicated mode
+# TODO((https://github.com/oxidecomputer/omicron/issues/4000)): Remove single node mode once all racks are running in replicated mode
 if $single_node
 then
   command+=(
@@ -94,8 +94,9 @@ else
   fi
 
   # Identify the node type this is as this will influence how the config is constructed
-  # TODO: There are probably much better ways to do this service discovery, but this works
-  # for now. The services contain the same IDs as the hostnames.
+  # TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are probably much 
+  # better ways to do this service discovery, but this works for now. 
+  # The services contain the same IDs as the hostnames.
   CLICKHOUSE_SVC="$(zonename | tr -dc [:digit:])"
   REPLICA_IDENTIFIER_01="$( echo "${REPLICA_HOST_01}" | tr -dc [:digit:])"
   REPLICA_IDENTIFIER_02="$( echo "${REPLICA_HOST_02}" | tr -dc [:digit:])"

smf/clickhouse_keeper/method_script.sh

@@ -69,7 +69,7 @@ KEEPER_ID_02="$( echo "${KEEPER_HOST_02}" | tr -dc [:digit:] | cut -c1-7)"
 KEEPER_ID_03="$( echo "${KEEPER_HOST_03}" | tr -dc [:digit:] | cut -c1-7)"
 
 # Identify the node type this is as this will influence how the config is constructed
-# TODO: There are probably much better ways to do this service name lookup, but this works
+# TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are probably much better ways to do this service name lookup, but this works
 # for now. The services contain the same IDs as the hostnames.
 KEEPER_SVC="$(zonename | tr -dc [:digit:] | cut -c1-7)"
 if [[ $KEEPER_ID_01 == $KEEPER_SVC ]]

test-utils/src/dev/clickhouse.rs

@@ -20,7 +20,8 @@ use tokio::{
 use crate::dev::poll;
 
 // Timeout used when starting up ClickHouse subprocess.
-const CLICKHOUSE_TIMEOUT: Duration = Duration::from_secs(60);
+// build-and-test (ubuntu-20.04) needs a little longer to get going 
+const CLICKHOUSE_TIMEOUT: Duration = Duration::from_secs(90);
 
 /// A `ClickHouseInstance` is used to start and manage a ClickHouse single node server process.
 #[derive(Debug)]