From b4fa8755d1c12aa9d55cffe2054c6a26378ad3a0 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 19 Nov 2024 10:55:26 -0700 Subject: [PATCH] Single node clickhouse init (#6903) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Depends on #6894, replaces #6878, fixes #6826 on the server side. Adds a new Dropshot server, `clickhouse-admin-single`, analogous to `clickhouse-admin-keeper` and `clickhouse-admin-server` (which were split off from `clickhouse-admin` in #6837). Its sole purpose is to initialize the single-node ClickHouse database with the current Oximeter schema. We use a single in-memory lock (a `tokio::sync::Mutex`) to serialize initialization requests. Multi-node ClickHouse clusters will need something analogous as a follow-up. --------- Co-authored-by: Andrew J. Stone Co-authored-by: Karen Cárcamo --- Cargo.lock | 29 ++++++-- Cargo.toml | 3 + clickhouse-admin/Cargo.toml | 2 +- clickhouse-admin/api/src/lib.rs | 23 +++++- .../src/bin/clickhouse-admin-single.rs | 72 +++++++++++++++++++ clickhouse-admin/src/context.rs | 21 ++++++ clickhouse-admin/src/http_entrypoints.rs | 63 ++++++++++++++-- clickhouse-admin/src/lib.rs | 44 ++++++++++-- .../clickhouse-admin-single-client/Cargo.toml | 18 +++++ .../clickhouse-admin-single-client/src/lib.rs | 26 +++++++ dev-tools/ls-apis/api-manifest.toml | 9 +++ dev-tools/ls-apis/tests/api_dependencies.out | 3 + dev-tools/openapi-manager/src/spec.rs | 11 +++ nexus/Cargo.toml | 1 + nexus/reconfigurator/execution/Cargo.toml | 1 + .../execution/src/clickhouse.rs | 39 ++++++++++ nexus/reconfigurator/execution/src/lib.rs | 32 +++++++++ openapi/clickhouse-admin-single.json | 66 +++++++++++++++++ package-manifest.toml | 8 +-- sled-agent/src/services.rs | 9 +-- smf/clickhouse-admin-single/config.toml | 10 +++ smf/clickhouse-admin-single/manifest.xml | 46 ++++++++++++ 22 files changed, 509 insertions(+), 27 deletions(-) create mode 100644 clickhouse-admin/src/bin/clickhouse-admin-single.rs create mode 100644 clients/clickhouse-admin-single-client/Cargo.toml create mode 100644 clients/clickhouse-admin-single-client/src/lib.rs create mode 100644 openapi/clickhouse-admin-single.json create mode 100644 smf/clickhouse-admin-single/config.toml create mode 100644 smf/clickhouse-admin-single/manifest.xml diff --git a/Cargo.lock b/Cargo.lock index 69cd1923e5..727ec68f27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1375,6 +1375,21 @@ dependencies = [ "slog", ] +[[package]] +name = "clickhouse-admin-single-client" +version = "0.1.0" +dependencies = [ + "chrono", + "clickhouse-admin-types", + "omicron-uuid-kinds", + "omicron-workspace-hack", + "progenitor", + "reqwest", + "schemars", + "serde", + "slog", +] + [[package]] name = "clickhouse-admin-test-utils" version = "0.1.0" @@ -5967,6 +5982,7 @@ dependencies = [ "chrono", "clickhouse-admin-keeper-client", "clickhouse-admin-server-client", + "clickhouse-admin-single-client", "clickhouse-admin-types", "cockroach-admin-client", "diesel", @@ -6846,6 +6862,7 @@ dependencies = [ "clap", "clickhouse-admin-keeper-client", "clickhouse-admin-server-client", + "clickhouse-admin-single-client", "cockroach-admin-client", "criterion", "crucible-agent-client", @@ -7543,9 +7560,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.68" +version = "0.10.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -7575,9 +7592,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.104" +version = "0.9.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" dependencies = [ "cc", "libc", @@ -11423,9 +11440,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.43" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6" +checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" dependencies = [ "filetime", "libc", diff --git a/Cargo.toml b/Cargo.toml index 797f7394d8..c5b7a2a4a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "clients/bootstrap-agent-client", "clients/clickhouse-admin-keeper-client", "clients/clickhouse-admin-server-client", + "clients/clickhouse-admin-single-client", "clients/cockroach-admin-client", "clients/ddm-admin-client", "clients/dns-service-client", @@ -137,6 +138,7 @@ default-members = [ "clients/bootstrap-agent-client", "clients/clickhouse-admin-keeper-client", "clients/clickhouse-admin-server-client", + "clients/clickhouse-admin-single-client", "clients/cockroach-admin-client", "clients/ddm-admin-client", "clients/dns-service-client", @@ -329,6 +331,7 @@ clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] } clickhouse-admin-api = { path = "clickhouse-admin/api" } clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-client" } clickhouse-admin-server-client = { path = "clients/clickhouse-admin-server-client" } +clickhouse-admin-single-client = { path = "clients/clickhouse-admin-single-client" } clickhouse-admin-types = { path = "clickhouse-admin/types" } clickhouse-admin-test-utils = { path = "clickhouse-admin/test-utils" } clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "a1b342c2558e835d09e6e39a40d3de798a29c2f" } diff --git a/clickhouse-admin/Cargo.toml b/clickhouse-admin/Cargo.toml index 3a8d45dd64..80b080b2ff 100644 --- a/clickhouse-admin/Cargo.toml +++ b/clickhouse-admin/Cargo.toml @@ -16,6 +16,7 @@ http.workspace = true illumos-utils.workspace = true omicron-common.workspace = true omicron-uuid-kinds.workspace = true +oximeter-db.workspace = true schemars.workspace = true slog.workspace = true slog-async.workspace = true @@ -35,7 +36,6 @@ clickhouse-admin-test-utils.workspace = true dropshot.workspace = true expectorate.workspace = true omicron-test-utils.workspace = true -oximeter-db.workspace = true oximeter-test-utils.workspace = true openapi-lint.workspace = true openapiv3.workspace = true diff --git a/clickhouse-admin/api/src/lib.rs b/clickhouse-admin/api/src/lib.rs index 8b7dff60f3..398fb30f06 100644 --- a/clickhouse-admin/api/src/lib.rs +++ b/clickhouse-admin/api/src/lib.rs @@ -8,7 +8,8 @@ use clickhouse_admin_types::{ ServerConfigurableSettings, }; use dropshot::{ - HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, + HttpError, HttpResponseCreated, HttpResponseOk, + HttpResponseUpdatedNoContent, RequestContext, TypedBody, }; /// API interface for our clickhouse-admin-keeper server @@ -116,3 +117,23 @@ pub trait ClickhouseAdminServerApi { rqctx: RequestContext, ) -> Result>, HttpError>; } + +/// API interface for our clickhouse-admin-single server +/// +/// The single-node server is distinct from the both the multi-node servers +/// and its keepers. The sole purpose of this API is to serialize database +/// initialization requests from reconfigurator execution. Multi-node clusters +/// must provide a similar interface via [`ClickhouseAdminServerApi`]. +#[dropshot::api_description] +pub trait ClickhouseAdminSingleApi { + type Context; + + /// Idempotently initialize a single-node ClickHouse database. + #[endpoint { + method = PUT, + path = "/init" + }] + async fn init_db( + rqctx: RequestContext, + ) -> Result; +} diff --git a/clickhouse-admin/src/bin/clickhouse-admin-single.rs b/clickhouse-admin/src/bin/clickhouse-admin-single.rs new file mode 100644 index 0000000000..bc7097f980 --- /dev/null +++ b/clickhouse-admin/src/bin/clickhouse-admin-single.rs @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Single-node ClickHouse database admin binary. + +use anyhow::anyhow; +use camino::Utf8PathBuf; +use clap::Parser; +use omicron_clickhouse_admin::{ClickhouseCli, Config}; +use omicron_common::cmd::fatal; +use omicron_common::cmd::CmdError; +use std::net::{SocketAddr, SocketAddrV6}; + +#[derive(Debug, Parser)] +#[clap( + name = "clickhouse-admin-single", + about = "Single-node ClickHouse admin server" +)] +enum Args { + /// Start the single-node ClickHouse admin server + Run { + /// Address on which this server should run + #[clap(long, short = 'a', action)] + http_address: SocketAddrV6, + + /// Path to the server configuration file + #[clap(long, short, action)] + config: Utf8PathBuf, + + /// Address of the ClickHouse single-node database server + #[clap(long, short = 'l', action)] + listen_address: SocketAddrV6, + + /// Path to the clickhouse binary + #[clap(long, short, action)] + binary_path: Utf8PathBuf, + }, +} + +#[tokio::main] +async fn main() { + if let Err(err) = main_impl().await { + fatal(err); + } +} + +async fn main_impl() -> Result<(), CmdError> { + let args = Args::parse(); + + match args { + Args::Run { http_address, config, listen_address, binary_path } => { + let mut config = Config::from_file(&config) + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + config.dropshot.bind_address = SocketAddr::V6(http_address); + let clickhouse_cli = + ClickhouseCli::new(binary_path, listen_address); + + let server = omicron_clickhouse_admin::start_single_admin_server( + clickhouse_cli, + config, + ) + .await + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + server.await.map_err(|err| { + CmdError::Failure(anyhow!( + "server failed after starting: {err}" + )) + }) + } + } +} diff --git a/clickhouse-admin/src/context.rs b/clickhouse-admin/src/context.rs index 665d19528a..f423dafe1f 100644 --- a/clickhouse-admin/src/context.rs +++ b/clickhouse-admin/src/context.rs @@ -4,6 +4,8 @@ use crate::{ClickhouseCli, Clickward}; use slog::Logger; +use std::sync::Arc; +use tokio::sync::Mutex; pub struct ServerContext { clickward: Clickward, @@ -28,3 +30,22 @@ impl ServerContext { &self.clickhouse_cli } } + +pub struct SingleServerContext { + clickhouse_cli: ClickhouseCli, + initialization_lock: Arc>, +} + +impl SingleServerContext { + pub fn new(clickhouse_cli: ClickhouseCli) -> Self { + Self { clickhouse_cli, initialization_lock: Arc::new(Mutex::new(())) } + } + + pub fn clickhouse_cli(&self) -> &ClickhouseCli { + &self.clickhouse_cli + } + + pub fn initialization_lock(&self) -> Arc> { + self.initialization_lock.clone() + } +} diff --git a/clickhouse-admin/src/http_entrypoints.rs b/clickhouse-admin/src/http_entrypoints.rs index dc28f8c6af..4380318476 100644 --- a/clickhouse-admin/src/http_entrypoints.rs +++ b/clickhouse-admin/src/http_entrypoints.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::context::ServerContext; +use crate::context::{ServerContext, SingleServerContext}; use clickhouse_admin_api::*; use clickhouse_admin_types::{ ClickhouseKeeperClusterMembership, DistributedDdlQueue, KeeperConf, @@ -10,23 +10,32 @@ use clickhouse_admin_types::{ ServerConfigurableSettings, }; use dropshot::{ - HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, + ApiDescription, HttpError, HttpResponseCreated, HttpResponseOk, + HttpResponseUpdatedNoContent, RequestContext, TypedBody, }; use illumos_utils::svcadm::Svcadm; +use omicron_common::address::CLICKHOUSE_TCP_PORT; +use oximeter_db::{Client as OximeterClient, OXIMETER_VERSION}; +use slog::debug; +use std::net::SocketAddrV6; use std::sync::Arc; -type ClickhouseApiDescription = dropshot::ApiDescription>; - -pub fn clickhouse_admin_server_api() -> ClickhouseApiDescription { +pub fn clickhouse_admin_server_api() -> ApiDescription> { clickhouse_admin_server_api_mod::api_description::() .expect("registered entrypoints") } -pub fn clickhouse_admin_keeper_api() -> ClickhouseApiDescription { +pub fn clickhouse_admin_keeper_api() -> ApiDescription> { clickhouse_admin_keeper_api_mod::api_description::() .expect("registered entrypoints") } +pub fn clickhouse_admin_single_api() -> ApiDescription> +{ + clickhouse_admin_single_api_mod::api_description::() + .expect("registered entrypoints") +} + enum ClickhouseAdminServerImpl {} impl ClickhouseAdminServerApi for ClickhouseAdminServerImpl { @@ -110,3 +119,45 @@ impl ClickhouseAdminKeeperApi for ClickhouseAdminKeeperImpl { Ok(HttpResponseOk(output)) } } + +enum ClickhouseAdminSingleImpl {} + +impl ClickhouseAdminSingleApi for ClickhouseAdminSingleImpl { + type Context = Arc; + + async fn init_db( + rqctx: RequestContext, + ) -> Result { + let log = &rqctx.log; + let ctx = rqctx.context(); + let http_address = ctx.clickhouse_cli().listen_address; + let native_address = + SocketAddrV6::new(*http_address.ip(), CLICKHOUSE_TCP_PORT, 0, 0); + let client = OximeterClient::new( + http_address.into(), + native_address.into(), + log, + ); + debug!( + log, + "initializing single-node ClickHouse \ + at {http_address} to version {OXIMETER_VERSION}" + ); + + // Database initialization is idempotent, but not concurrency-safe. + // Use a mutex to serialize requests. + let lock = ctx.initialization_lock(); + let _guard = lock.lock().await; + client + .initialize_db_with_version(false, OXIMETER_VERSION) + .await + .map_err(|e| { + HttpError::for_internal_error(format!( + "can't initialize single-node ClickHouse \ + at {http_address} to version {OXIMETER_VERSION}: {e}", + )) + })?; + + Ok(HttpResponseUpdatedNoContent()) + } +} diff --git a/clickhouse-admin/src/lib.rs b/clickhouse-admin/src/lib.rs index 76e55e1e2b..0a79978e32 100644 --- a/clickhouse-admin/src/lib.rs +++ b/clickhouse-admin/src/lib.rs @@ -2,7 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use context::ServerContext; +use context::{ServerContext, SingleServerContext}; +use dropshot::HttpServer; use omicron_common::FileKv; use slog::{debug, error, Drain}; use slog_dtrace::ProbeRegistration; @@ -30,15 +31,13 @@ pub enum StartError { InitializeHttpServer(#[source] dropshot::BuildError), } -pub type Server = dropshot::HttpServer>; - /// Start the dropshot server for `clickhouse-admin-server` which /// manages clickhouse replica servers. pub async fn start_server_admin_server( clickward: Clickward, clickhouse_cli: ClickhouseCli, server_config: Config, -) -> Result { +) -> Result>, StartError> { let (drain, registration) = slog_dtrace::with_drain( server_config .log @@ -80,7 +79,7 @@ pub async fn start_keeper_admin_server( clickward: Clickward, clickhouse_cli: ClickhouseCli, server_config: Config, -) -> Result { +) -> Result>, StartError> { let (drain, registration) = slog_dtrace::with_drain( server_config .log @@ -115,3 +114,38 @@ pub async fn start_keeper_admin_server( .start() .map_err(StartError::InitializeHttpServer) } + +/// Start the dropshot server for `clickhouse-admin-single` which +/// manages a single-node ClickHouse database. +pub async fn start_single_admin_server( + clickhouse_cli: ClickhouseCli, + server_config: Config, +) -> Result>, StartError> { + let (drain, registration) = slog_dtrace::with_drain( + server_config + .log + .to_logger("clickhouse-admin-single") + .map_err(StartError::InitializeLogger)?, + ); + let log = slog::Logger::root(drain.fuse(), slog::o!(FileKv)); + match registration { + ProbeRegistration::Success => { + debug!(log, "registered DTrace probes"); + } + ProbeRegistration::Failed(err) => { + let err = StartError::RegisterDtraceProbes(err); + error!(log, "failed to register DTrace probes"; &err); + return Err(err); + } + } + + let context = SingleServerContext::new(clickhouse_cli); + dropshot::ServerBuilder::new( + http_entrypoints::clickhouse_admin_single_api(), + Arc::new(context), + log.new(slog::o!("component" => "dropshot")), + ) + .config(server_config.dropshot) + .start() + .map_err(StartError::InitializeHttpServer) +} diff --git a/clients/clickhouse-admin-single-client/Cargo.toml b/clients/clickhouse-admin-single-client/Cargo.toml new file mode 100644 index 0000000000..e8b2abf860 --- /dev/null +++ b/clients/clickhouse-admin-single-client/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "clickhouse-admin-single-client" +version = "0.1.0" +edition = "2021" + +[dependencies] +chrono.workspace = true +clickhouse-admin-types.workspace = true +omicron-uuid-kinds.workspace = true +progenitor.workspace = true +reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } +schemars.workspace = true +serde.workspace = true +slog.workspace = true +omicron-workspace-hack.workspace = true + +[lints] +workspace = true diff --git a/clients/clickhouse-admin-single-client/src/lib.rs b/clients/clickhouse-admin-single-client/src/lib.rs new file mode 100644 index 0000000000..1197310800 --- /dev/null +++ b/clients/clickhouse-admin-single-client/src/lib.rs @@ -0,0 +1,26 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! API for the clickhouse-admin-single server to manage a single-node +//! ClickHouse database. + +progenitor::generate_api!( + spec = "../../openapi/clickhouse-admin-single.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + derives = [schemars::JsonSchema], + replace = { + TypedUuidForOmicronZoneKind = omicron_uuid_kinds::OmicronZoneUuid, + ServerConfigurableSettings = clickhouse_admin_types::ServerConfigurableSettings, + } +); diff --git a/dev-tools/ls-apis/api-manifest.toml b/dev-tools/ls-apis/api-manifest.toml index ab5dd4dec8..e3dbeed604 100644 --- a/dev-tools/ls-apis/api-manifest.toml +++ b/dev-tools/ls-apis/api-manifest.toml @@ -156,6 +156,15 @@ This is the server running inside multi-node Clickhouse server zones that's \ responsible for local configuration and monitoring. """ +[[apis]] +client_package_name = "clickhouse-admin-single-client" +label = "Clickhouse Single-Node Cluster Admin" +server_package_name = "clickhouse-admin-api" +notes = """ +This is the server running inside single-node Clickhouse server zones that's \ +responsible for local configuration and monitoring. +""" + [[apis]] client_package_name = "cockroach-admin-client" label = "CockroachDB Cluster Admin" diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index aee8cd7a70..7da9613014 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -8,6 +8,9 @@ Clickhouse Cluster Admin for Keepers (client: clickhouse-admin-keeper-client) Clickhouse Cluster Admin for Servers (client: clickhouse-admin-server-client) consumed by: omicron-nexus (omicron/nexus) via 3 paths +Clickhouse Single-Node Cluster Admin (client: clickhouse-admin-single-client) + consumed by: omicron-nexus (omicron/nexus) via 2 paths + CockroachDB Cluster Admin (client: cockroach-admin-client) consumed by: omicron-nexus (omicron/nexus) via 2 paths diff --git a/dev-tools/openapi-manager/src/spec.rs b/dev-tools/openapi-manager/src/spec.rs index 81ff54f409..57cbbea5da 100644 --- a/dev-tools/openapi-manager/src/spec.rs +++ b/dev-tools/openapi-manager/src/spec.rs @@ -48,6 +48,17 @@ pub fn all_apis() -> Vec { filename: "clickhouse-admin-server.json", extra_validation: None, }, + ApiSpec { + title: "ClickHouse Single-Node Admin Server API", + version: semver::Version::new(0, 0, 1), + description: "API for interacting with the Oxide \ + control plane's single-node ClickHouse database", + boundary: ApiBoundary::Internal, + api_description: + clickhouse_admin_api::clickhouse_admin_single_api_mod::stub_api_description, + filename: "clickhouse-admin-single.json", + extra_validation: None, + }, ApiSpec { title: "CockroachDB Cluster Admin API", version: semver::Version::new(0, 0, 1), diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index d430009360..6ba66e058f 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -24,6 +24,7 @@ clap.workspace = true chrono.workspace = true clickhouse-admin-keeper-client.workspace = true clickhouse-admin-server-client.workspace = true +clickhouse-admin-single-client.workspace = true cockroach-admin-client.workspace = true crucible-agent-client.workspace = true crucible-pantry-client.workspace = true diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml index 21d861ef51..e07e4f5f75 100644 --- a/nexus/reconfigurator/execution/Cargo.toml +++ b/nexus/reconfigurator/execution/Cargo.toml @@ -14,6 +14,7 @@ anyhow.workspace = true camino.workspace = true clickhouse-admin-keeper-client.workspace = true clickhouse-admin-server-client.workspace = true +clickhouse-admin-single-client.workspace = true clickhouse-admin-types.workspace = true cockroach-admin-client.workspace = true chrono.workspace = true diff --git a/nexus/reconfigurator/execution/src/clickhouse.rs b/nexus/reconfigurator/execution/src/clickhouse.rs index c7413c14f7..36e41aec1e 100644 --- a/nexus/reconfigurator/execution/src/clickhouse.rs +++ b/nexus/reconfigurator/execution/src/clickhouse.rs @@ -9,6 +9,7 @@ use anyhow::anyhow; use camino::Utf8PathBuf; use clickhouse_admin_keeper_client::Client as ClickhouseKeeperClient; use clickhouse_admin_server_client::Client as ClickhouseServerClient; +use clickhouse_admin_single_client::Client as ClickhouseSingleClient; use clickhouse_admin_types::ClickhouseHost; use clickhouse_admin_types::KeeperConfigurableSettings; use clickhouse_admin_types::KeeperSettings; @@ -19,6 +20,8 @@ use futures::future::Either; use futures::stream::FuturesUnordered; use futures::stream::StreamExt; use nexus_db_queries::context::OpContext; +use nexus_sled_agent_shared::inventory::OmicronZoneType; +use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::BlueprintZonesConfig; use nexus_types::deployment::ClickhouseClusterConfig; use omicron_common::address::CLICKHOUSE_ADMIN_PORT; @@ -160,6 +163,42 @@ pub(crate) async fn deploy_nodes( Ok(()) } +pub(crate) async fn deploy_single_node( + opctx: &OpContext, + zones: &BTreeMap, +) -> Result<(), anyhow::Error> { + if let Some(zone) = zones + .values() + .flat_map(|zones| { + zones + .to_omicron_zones_config(BlueprintZoneFilter::ShouldBeRunning) + .zones + .into_iter() + .find(|zone| { + matches!(zone.zone_type, OmicronZoneType::Clickhouse { .. }) + }) + }) + .next() + { + let admin_addr = SocketAddr::V6(SocketAddrV6::new( + zone.underlay_ip(), + CLICKHOUSE_ADMIN_PORT, + 0, + 0, + )); + let admin_url = format!("http://{admin_addr}"); + let log = opctx.log.new(slog::o!("admin_url" => admin_url.clone())); + let client = ClickhouseSingleClient::new(&admin_url, log.clone()); + client.init_db().await.map(|_| ()).map_err(|e| { + anyhow!( + "failed to initialize single-node clickhouse database: {e}", + ) + }) + } else { + Ok(()) + } +} + fn server_configs( zones: &BTreeMap, clickhouse_cluster_config: &ClickhouseClusterConfig, diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs index 48ab3996ff..570a3512eb 100644 --- a/nexus/reconfigurator/execution/src/lib.rs +++ b/nexus/reconfigurator/execution/src/lib.rs @@ -28,6 +28,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use tokio::sync::mpsc; use update_engine::merge_anyhow_list; +use update_engine::StepWarning; mod clickhouse; mod cockroachdb; @@ -182,6 +183,12 @@ pub async fn realize_blueprint_with_overrides( blueprint, ); + register_deploy_clickhouse_single_node_step( + &engine.for_component(ExecutionComponent::Clickhouse), + &opctx, + blueprint, + ); + let reassign_saga_output = register_reassign_sagas_step( &engine.for_component(ExecutionComponent::OmicronZones), &opctx, @@ -549,6 +556,31 @@ fn register_deploy_clickhouse_cluster_nodes_step<'a>( .register(); } +fn register_deploy_clickhouse_single_node_step<'a>( + registrar: &ComponentRegistrar<'_, 'a>, + opctx: &'a OpContext, + blueprint: &'a Blueprint, +) { + registrar + .new_step( + ExecutionStepId::Ensure, + "Deploy single-node clickhouse cluster", + move |_cx| async move { + if let Err(e) = clickhouse::deploy_single_node( + &opctx, + &blueprint.blueprint_zones, + ) + .await + { + StepWarning::new((), e.to_string()).into() + } else { + StepSuccess::new(()).into() + } + }, + ) + .register(); +} + #[derive(Debug)] struct ReassignSagaOutput { needs_saga_recovery: bool, diff --git a/openapi/clickhouse-admin-single.json b/openapi/clickhouse-admin-single.json new file mode 100644 index 0000000000..74763957ca --- /dev/null +++ b/openapi/clickhouse-admin-single.json @@ -0,0 +1,66 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "ClickHouse Single-Node Admin Server API", + "description": "API for interacting with the Oxide control plane's single-node ClickHouse database", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "0.0.1" + }, + "paths": { + "/init": { + "put": { + "summary": "Idempotently initialize a single-node ClickHouse database.", + "operationId": "init_db", + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/package-manifest.toml b/package-manifest.toml index e1d6d31c53..54e1236fe5 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -155,15 +155,14 @@ output.type = "zone" output.intermediate_only = true [package.clickhouse] -# This service runs a single-node ClickHouse server. +# This service runs a single-node ClickHouse server +# and a small administrative Dropshot server. service_name = "clickhouse" only_for_targets.image = "standard" source.type = "composite" source.packages = [ "clickhouse_svc.tar.gz", "internal-dns-cli.tar.gz", - # TODO: This package is for solely for testing purposes. - # Remove once replicated clickhouse is up and running. "omicron-clickhouse-admin.tar.gz", "zone-setup.tar.gz", "zone-network-install.tar.gz" @@ -179,6 +178,7 @@ source.paths = [ { from = "smf/clickhouse/manifest.xml", to = "/var/svc/manifest/site/clickhouse/manifest.xml" }, { from = "smf/clickhouse/method_script.sh", to = "/opt/oxide/lib/svc/manifest/clickhouse.sh" }, { from = "smf/clickhouse/config.xml", to = "/opt/oxide/clickhouse/config.xml" }, + { from = "smf/clickhouse-admin-single", to = "/var/svc/manifest/site/clickhouse-admin-single" }, ] output.type = "zone" output.intermediate_only = true @@ -248,7 +248,7 @@ setup_hint = "Run `cargo xtask download clickhouse` to download the necessary bi service_name = "omicron-clickhouse-admin" only_for_targets.image = "standard" source.type = "local" -source.rust.binary_names = ["clickhouse-admin-keeper", "clickhouse-admin-server"] +source.rust.binary_names = ["clickhouse-admin-keeper", "clickhouse-admin-server", "clickhouse-admin-single"] source.rust.release = true # We specifically put the smf manifests with their corresponding container zones # so that both servers aren't started simultaneously diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index a69dd4f29e..f0a0da4675 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1619,10 +1619,11 @@ impl ServiceManager { CLICKHOUSE_BINARY, ); let clickhouse_admin_service = - ServiceBuilder::new("oxide/clickhouse-admin").add_instance( - ServiceInstanceBuilder::new("default") - .add_property_group(clickhouse_admin_config), - ); + ServiceBuilder::new("oxide/clickhouse-admin-single") + .add_instance( + ServiceInstanceBuilder::new("default") + .add_property_group(clickhouse_admin_config), + ); let profile = ProfileBuilder::new("omicron") .add_service(nw_setup_service) diff --git a/smf/clickhouse-admin-single/config.toml b/smf/clickhouse-admin-single/config.toml new file mode 100644 index 0000000000..86ee2c5d4b --- /dev/null +++ b/smf/clickhouse-admin-single/config.toml @@ -0,0 +1,10 @@ +[dropshot] +# 1 MiB; we don't expect any requests of more than nominal size. +request_body_max_bytes = 1048576 + +[log] +# Show log messages of this level and more severe +level = "info" +mode = "file" +path = "/dev/stdout" +if_exists = "append" diff --git a/smf/clickhouse-admin-single/manifest.xml b/smf/clickhouse-admin-single/manifest.xml new file mode 100644 index 0000000000..f259020145 --- /dev/null +++ b/smf/clickhouse-admin-single/manifest.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +