Single node clickhouse init (#6903)

Depends on #6894, replaces #6878, fixes #6826 on the server side. Adds a new Dropshot server, `clickhouse-admin-single`, analogous to `clickhouse-admin-keeper` and `clickhouse-admin-server` (which were split off from `clickhouse-admin` in #6837). Its sole purpose is to initialize the single-node ClickHouse database with the current Oximeter schema. We use a single in-memory lock (a `tokio::sync::Mutex`) to serialize initialization requests. Multi-node ClickHouse clusters will need something analogous as a follow-up. --------- Co-authored-by: Andrew J. Stone <[email protected]> Co-authored-by: Karen Cárcamo <[email protected]>
oxidecomputer · Nov 19, 2024 · b4fa875 · b4fa875
1 parent 574b697
commit b4fa875
Show file tree

Hide file tree

Showing 22 changed files with 509 additions and 27 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -9,6 +9,7 @@ members = [
     "clients/bootstrap-agent-client",
     "clients/clickhouse-admin-keeper-client",
     "clients/clickhouse-admin-server-client",
+    "clients/clickhouse-admin-single-client",
     "clients/cockroach-admin-client",
     "clients/ddm-admin-client",
     "clients/dns-service-client",
@@ -137,6 +138,7 @@ default-members = [
     "clients/bootstrap-agent-client",
     "clients/clickhouse-admin-keeper-client",
     "clients/clickhouse-admin-server-client",
+    "clients/clickhouse-admin-single-client",
     "clients/cockroach-admin-client",
     "clients/ddm-admin-client",
     "clients/dns-service-client",
@@ -329,6 +331,7 @@ clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] }
 clickhouse-admin-api = { path = "clickhouse-admin/api" }
 clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-client" }
 clickhouse-admin-server-client = { path = "clients/clickhouse-admin-server-client" }
+clickhouse-admin-single-client = { path = "clients/clickhouse-admin-single-client" }
 clickhouse-admin-types = { path = "clickhouse-admin/types" }
 clickhouse-admin-test-utils = { path = "clickhouse-admin/test-utils" }
 clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "a1b342c2558e835d09e6e39a40d3de798a29c2f" }

diff --git a/clickhouse-admin/Cargo.toml b/clickhouse-admin/Cargo.toml
@@ -16,6 +16,7 @@ http.workspace = true
 illumos-utils.workspace = true
 omicron-common.workspace = true
 omicron-uuid-kinds.workspace = true
+oximeter-db.workspace = true
 schemars.workspace = true
 slog.workspace = true
 slog-async.workspace = true
@@ -35,7 +36,6 @@ clickhouse-admin-test-utils.workspace = true
 dropshot.workspace = true
 expectorate.workspace = true
 omicron-test-utils.workspace = true
-oximeter-db.workspace = true
 oximeter-test-utils.workspace = true
 openapi-lint.workspace = true
 openapiv3.workspace = true

diff --git a/clickhouse-admin/api/src/lib.rs b/clickhouse-admin/api/src/lib.rs
@@ -8,7 +8,8 @@ use clickhouse_admin_types::{
     ServerConfigurableSettings,
 };
 use dropshot::{
-    HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody,
+    HttpError, HttpResponseCreated, HttpResponseOk,
+    HttpResponseUpdatedNoContent, RequestContext, TypedBody,
 };
 
 /// API interface for our clickhouse-admin-keeper server
@@ -116,3 +117,23 @@ pub trait ClickhouseAdminServerApi {
         rqctx: RequestContext<Self::Context>,
     ) -> Result<HttpResponseOk<Vec<DistributedDdlQueue>>, HttpError>;
 }
+
+/// API interface for our clickhouse-admin-single server
+///
+/// The single-node server is distinct from the both the multi-node servers
+/// and its keepers. The sole purpose of this API is to serialize database
+/// initialization requests from reconfigurator execution. Multi-node clusters
+/// must provide a similar interface via [`ClickhouseAdminServerApi`].
+#[dropshot::api_description]
+pub trait ClickhouseAdminSingleApi {
+    type Context;
+
+    /// Idempotently initialize a single-node ClickHouse database.
+    #[endpoint {
+        method = PUT,
+        path = "/init"
+    }]
+    async fn init_db(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+}
diff --git a/clickhouse-admin/src/bin/clickhouse-admin-single.rs b/clickhouse-admin/src/bin/clickhouse-admin-single.rs
@@ -0,0 +1,72 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Single-node ClickHouse database admin binary.
+
+use anyhow::anyhow;
+use camino::Utf8PathBuf;
+use clap::Parser;
+use omicron_clickhouse_admin::{ClickhouseCli, Config};
+use omicron_common::cmd::fatal;
+use omicron_common::cmd::CmdError;
+use std::net::{SocketAddr, SocketAddrV6};
+
+#[derive(Debug, Parser)]
+#[clap(
+    name = "clickhouse-admin-single",
+    about = "Single-node ClickHouse admin server"
+)]
+enum Args {
+    /// Start the single-node ClickHouse admin server
+    Run {
+        /// Address on which this server should run
+        #[clap(long, short = 'a', action)]
+        http_address: SocketAddrV6,
+
+        /// Path to the server configuration file
+        #[clap(long, short, action)]
+        config: Utf8PathBuf,
+
+        /// Address of the ClickHouse single-node database server
+        #[clap(long, short = 'l', action)]
+        listen_address: SocketAddrV6,
+
+        /// Path to the clickhouse binary
+        #[clap(long, short, action)]
+        binary_path: Utf8PathBuf,
+    },
+}
+
+#[tokio::main]
+async fn main() {
+    if let Err(err) = main_impl().await {
+        fatal(err);
+    }
+}
+
+async fn main_impl() -> Result<(), CmdError> {
+    let args = Args::parse();
+
+    match args {
+        Args::Run { http_address, config, listen_address, binary_path } => {
+            let mut config = Config::from_file(&config)
+                .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+            config.dropshot.bind_address = SocketAddr::V6(http_address);
+            let clickhouse_cli =
+                ClickhouseCli::new(binary_path, listen_address);
+
+            let server = omicron_clickhouse_admin::start_single_admin_server(
+                clickhouse_cli,
+                config,
+            )
+            .await
+            .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+            server.await.map_err(|err| {
+                CmdError::Failure(anyhow!(
+                    "server failed after starting: {err}"
+                ))
+            })
+        }
+    }
+}
diff --git a/clickhouse-admin/src/context.rs b/clickhouse-admin/src/context.rs
@@ -4,6 +4,8 @@
 
 use crate::{ClickhouseCli, Clickward};
 use slog::Logger;
+use std::sync::Arc;
+use tokio::sync::Mutex;
 
 pub struct ServerContext {
     clickward: Clickward,
@@ -28,3 +30,22 @@ impl ServerContext {
         &self.clickhouse_cli
     }
 }
+
+pub struct SingleServerContext {
+    clickhouse_cli: ClickhouseCli,
+    initialization_lock: Arc<Mutex<()>>,
+}
+
+impl SingleServerContext {
+    pub fn new(clickhouse_cli: ClickhouseCli) -> Self {
+        Self { clickhouse_cli, initialization_lock: Arc::new(Mutex::new(())) }
+    }
+
+    pub fn clickhouse_cli(&self) -> &ClickhouseCli {
+        &self.clickhouse_cli
+    }
+
+    pub fn initialization_lock(&self) -> Arc<Mutex<()>> {
+        self.initialization_lock.clone()
+    }
+}
diff --git a/clickhouse-admin/src/http_entrypoints.rs b/clickhouse-admin/src/http_entrypoints.rs
@@ -2,31 +2,40 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use crate::context::ServerContext;
+use crate::context::{ServerContext, SingleServerContext};
 use clickhouse_admin_api::*;
 use clickhouse_admin_types::{
     ClickhouseKeeperClusterMembership, DistributedDdlQueue, KeeperConf,
     KeeperConfig, KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig,
     ServerConfigurableSettings,
 };
 use dropshot::{
-    HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody,
+    ApiDescription, HttpError, HttpResponseCreated, HttpResponseOk,
+    HttpResponseUpdatedNoContent, RequestContext, TypedBody,
 };
 use illumos_utils::svcadm::Svcadm;
+use omicron_common::address::CLICKHOUSE_TCP_PORT;
+use oximeter_db::{Client as OximeterClient, OXIMETER_VERSION};
+use slog::debug;
+use std::net::SocketAddrV6;
 use std::sync::Arc;
 
-type ClickhouseApiDescription = dropshot::ApiDescription<Arc<ServerContext>>;
-
-pub fn clickhouse_admin_server_api() -> ClickhouseApiDescription {
+pub fn clickhouse_admin_server_api() -> ApiDescription<Arc<ServerContext>> {
     clickhouse_admin_server_api_mod::api_description::<ClickhouseAdminServerImpl>()
         .expect("registered entrypoints")
 }
 
-pub fn clickhouse_admin_keeper_api() -> ClickhouseApiDescription {
+pub fn clickhouse_admin_keeper_api() -> ApiDescription<Arc<ServerContext>> {
     clickhouse_admin_keeper_api_mod::api_description::<ClickhouseAdminKeeperImpl>()
         .expect("registered entrypoints")
 }
 
+pub fn clickhouse_admin_single_api() -> ApiDescription<Arc<SingleServerContext>>
+{
+    clickhouse_admin_single_api_mod::api_description::<ClickhouseAdminSingleImpl>()
+        .expect("registered entrypoints")
+}
+
 enum ClickhouseAdminServerImpl {}
 
 impl ClickhouseAdminServerApi for ClickhouseAdminServerImpl {
@@ -110,3 +119,45 @@ impl ClickhouseAdminKeeperApi for ClickhouseAdminKeeperImpl {
         Ok(HttpResponseOk(output))
     }
 }
+
+enum ClickhouseAdminSingleImpl {}
+
+impl ClickhouseAdminSingleApi for ClickhouseAdminSingleImpl {
+    type Context = Arc<SingleServerContext>;
+
+    async fn init_db(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let log = &rqctx.log;
+        let ctx = rqctx.context();
+        let http_address = ctx.clickhouse_cli().listen_address;
+        let native_address =
+            SocketAddrV6::new(*http_address.ip(), CLICKHOUSE_TCP_PORT, 0, 0);
+        let client = OximeterClient::new(
+            http_address.into(),
+            native_address.into(),
+            log,
+        );
+        debug!(
+            log,
+            "initializing single-node ClickHouse \
+             at {http_address} to version {OXIMETER_VERSION}"
+        );
+
+        // Database initialization is idempotent, but not concurrency-safe.
+        // Use a mutex to serialize requests.
+        let lock = ctx.initialization_lock();
+        let _guard = lock.lock().await;
+        client
+            .initialize_db_with_version(false, OXIMETER_VERSION)
+            .await
+            .map_err(|e| {
+                HttpError::for_internal_error(format!(
+                    "can't initialize single-node ClickHouse \
+                     at {http_address} to version {OXIMETER_VERSION}: {e}",
+                ))
+            })?;
+
+        Ok(HttpResponseUpdatedNoContent())
+    }
+}