diff --git a/clickhouse-admin/api/src/lib.rs b/clickhouse-admin/api/src/lib.rs index 7e66c9ca06..0fe29fa006 100644 --- a/clickhouse-admin/api/src/lib.rs +++ b/clickhouse-admin/api/src/lib.rs @@ -4,7 +4,8 @@ use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; use clickhouse_admin_types::{ - KeeperConf, KeeperSettings, Lgif, RaftConfig, ServerSettings, + ClickhouseKeeperClusterMembership, KeeperConf, KeeperSettings, Lgif, + RaftConfig, ServerSettings, }; use dropshot::{ HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, @@ -84,4 +85,13 @@ pub trait ClickhouseAdminApi { async fn keeper_conf( rqctx: RequestContext, ) -> Result, HttpError>; + + /// Retrieve cluster membership information from a keeper node. + #[endpoint { + method = GET, + path = "/keeper/cluster-membership", + }] + async fn keeper_cluster_membership( + rqctx: RequestContext, + ) -> Result, HttpError>; } diff --git a/clickhouse-admin/src/clickhouse_cli.rs b/clickhouse-admin/src/clickhouse_cli.rs index a84e2b3404..32afdc4ef8 100644 --- a/clickhouse-admin/src/clickhouse_cli.rs +++ b/clickhouse-admin/src/clickhouse_cli.rs @@ -4,11 +4,14 @@ use anyhow::Result; use camino::Utf8PathBuf; -use clickhouse_admin_types::{KeeperConf, Lgif, RaftConfig}; +use clickhouse_admin_types::{ + ClickhouseKeeperClusterMembership, KeeperConf, KeeperId, Lgif, RaftConfig, +}; use dropshot::HttpError; use illumos_utils::{output_to_exec_error, ExecutionError}; use slog::Logger; use slog_error_chain::{InlineErrorChain, SlogInlineError}; +use std::collections::BTreeSet; use std::ffi::OsStr; use std::io; use std::net::SocketAddrV6; @@ -102,6 +105,22 @@ impl ClickhouseCli { .await } + pub async fn keeper_cluster_membership( + &self, + ) -> Result { + let lgif_output = self.lgif().await?; + let conf_output = self.keeper_conf().await?; + let raft_output = self.raft_config().await?; + let raft_config: BTreeSet = + raft_output.keeper_servers.iter().map(|s| s.server_id).collect(); + + Ok(ClickhouseKeeperClusterMembership { + queried_keeper: conf_output.server_id, + leader_committed_log_index: lgif_output.leader_committed_log_idx, + raft_config, + }) + } + async fn keeper_client_non_interactive( &self, query: &str, diff --git a/clickhouse-admin/src/http_entrypoints.rs b/clickhouse-admin/src/http_entrypoints.rs index b1571f26c7..3b6f5a64ba 100644 --- a/clickhouse-admin/src/http_entrypoints.rs +++ b/clickhouse-admin/src/http_entrypoints.rs @@ -5,7 +5,9 @@ use crate::context::ServerContext; use clickhouse_admin_api::*; use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; -use clickhouse_admin_types::{KeeperConf, Lgif, RaftConfig}; +use clickhouse_admin_types::{ + ClickhouseKeeperClusterMembership, KeeperConf, Lgif, RaftConfig, +}; use dropshot::{ HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, }; @@ -71,4 +73,13 @@ impl ClickhouseAdminApi for ClickhouseAdminImpl { let output = ctx.clickhouse_cli().keeper_conf().await?; Ok(HttpResponseOk(output)) } + + async fn keeper_cluster_membership( + rqctx: RequestContext, + ) -> Result, HttpError> + { + let ctx = rqctx.context(); + let output = ctx.clickhouse_cli().keeper_cluster_membership().await?; + Ok(HttpResponseOk(output)) + } } diff --git a/clickhouse-admin/tests/integration_test.rs b/clickhouse-admin/tests/integration_test.rs index 79164b043f..fd706722a3 100644 --- a/clickhouse-admin/tests/integration_test.rs +++ b/clickhouse-admin/tests/integration_test.rs @@ -5,7 +5,10 @@ use anyhow::Context; use camino::Utf8PathBuf; use clickhouse_admin_types::config::ClickhouseHost; -use clickhouse_admin_types::{KeeperServerInfo, KeeperServerType, RaftConfig}; +use clickhouse_admin_types::{ + ClickhouseKeeperClusterMembership, KeeperId, KeeperServerInfo, + KeeperServerType, RaftConfig, +}; use clickward::{BasePorts, Deployment, DeploymentConfig}; use dropshot::test_util::log_prefix_for_test; use omicron_clickhouse_admin::ClickhouseCli; @@ -197,3 +200,83 @@ async fn test_keeper_conf_parsing() -> anyhow::Result<()> { logctx.cleanup_successful(); Ok(()) } + +#[tokio::test] +async fn test_keeper_cluster_membership() -> anyhow::Result<()> { + let logctx = test_setup_log("test_keeper_cluster_membership"); + let log = logctx.log.clone(); + + let (parent_dir, prefix) = log_prefix_for_test(logctx.test_name()); + let path = parent_dir.join(format!("{prefix}-oximeter-clickward-test")); + std::fs::create_dir(&path)?; + + // We spin up several replicated clusters and must use a + // separate set of ports in case the tests run concurrently. + let base_ports = BasePorts { + keeper: 30500, + raft: 30600, + clickhouse_tcp: 30700, + clickhouse_http: 30800, + clickhouse_interserver_http: 30900, + }; + + let config = DeploymentConfig { + path: path.clone(), + base_ports, + cluster_name: "oximeter_cluster".to_string(), + }; + + let mut deployment = Deployment::new(config); + + let num_keepers = 3; + let num_replicas = 1; + deployment + .generate_config(num_keepers, num_replicas) + .context("failed to generate config")?; + deployment.deploy().context("failed to deploy")?; + + wait_for_keepers( + &log, + &deployment, + (1..=num_keepers).map(clickward::KeeperId).collect(), + ) + .await?; + + let clickhouse_cli = ClickhouseCli::new( + Utf8PathBuf::from_str("clickhouse").unwrap(), + SocketAddrV6::new(Ipv6Addr::LOCALHOST, 30501, 0, 0), + ) + .with_log(log.clone()); + + let keeper_cluster_membership = + clickhouse_cli.keeper_cluster_membership().await.unwrap(); + + let mut raft_config = BTreeSet::new(); + + for i in 1..=num_keepers { + raft_config.insert(clickhouse_admin_types::KeeperId(i)); + } + + let expected_keeper_cluster_membership = + ClickhouseKeeperClusterMembership { + queried_keeper: KeeperId(1), + // This number is always different so we won't be testing it + leader_committed_log_index: 0, + raft_config, + }; + + assert_eq!( + keeper_cluster_membership.queried_keeper, + expected_keeper_cluster_membership.queried_keeper + ); + assert_eq!( + keeper_cluster_membership.raft_config, + expected_keeper_cluster_membership.raft_config + ); + + info!(&log, "Cleaning up test"); + deployment.teardown()?; + std::fs::remove_dir_all(path)?; + logctx.cleanup_successful(); + Ok(()) +} diff --git a/clickhouse-admin/types/src/lib.rs b/clickhouse-admin/types/src/lib.rs index a58f7b7cc4..03500678cb 100644 --- a/clickhouse-admin/types/src/lib.rs +++ b/clickhouse-admin/types/src/lib.rs @@ -900,6 +900,24 @@ impl KeeperConf { } } +/// The configuration of the clickhouse keeper raft cluster returned from a +/// single keeper node +/// +/// Each keeper is asked for its known raft configuration via `clickhouse-admin` +/// dropshot servers running in `ClickhouseKeeper` zones. state. We include the +/// leader committed log index known to the current keeper node (whether or not +/// it is the leader) to determine which configuration is newest. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct ClickhouseKeeperClusterMembership { + /// Keeper ID of the keeper being queried + pub queried_keeper: KeeperId, + /// Index of the last committed log entry from the leader's perspective + pub leader_committed_log_index: u64, + /// Keeper IDs of all keepers in the cluster + pub raft_config: BTreeSet, +} + #[cfg(test)] mod tests { use camino::Utf8PathBuf; diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs b/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs index 0941806fc3..257ab50bd1 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs @@ -5,12 +5,11 @@ //! A mechanism for allocating clickhouse keeper and server nodes for clustered //! clickhouse setups during blueprint planning -use clickhouse_admin_types::KeeperId; +use clickhouse_admin_types::{ClickhouseKeeperClusterMembership, KeeperId}; use nexus_types::deployment::{ Blueprint, BlueprintZoneFilter, BlueprintZoneType, BlueprintZonesConfig, ClickhouseClusterConfig, }; -use nexus_types::inventory::ClickhouseKeeperClusterMembership; use omicron_uuid_kinds::{OmicronZoneUuid, SledUuid}; use slog::{error, Logger}; use std::collections::{BTreeMap, BTreeSet}; diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 4f60f4d690..b15717798b 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -13,7 +13,7 @@ use crate::external_api::params::PhysicalDiskKind; use crate::external_api::params::UninitializedSledId; use chrono::DateTime; use chrono::Utc; -use clickhouse_admin_types::KeeperId; +use clickhouse_admin_types::ClickhouseKeeperClusterMembership; pub use gateway_client::types::PowerState; pub use gateway_client::types::RotImageError; pub use gateway_client::types::RotSlot; @@ -512,17 +512,3 @@ pub struct OmicronZonesFound { pub sled_id: SledUuid, pub zones: OmicronZonesConfig, } - -/// The configuration of the clickhouse keeper raft cluster returned from a -/// single keeper node -/// -/// Each keeper is asked for its known raft configuration via `clickhouse-admin` -/// dropshot servers running in `ClickhouseKeeper` zones. state. We include the -/// leader committed log index known to the current keeper node (whether or not -/// it is the leader) to determine which configuration is newest. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -pub struct ClickhouseKeeperClusterMembership { - pub queried_keeper: KeeperId, - pub leader_committed_log_index: u64, - pub raft_config: BTreeSet, -} diff --git a/openapi/clickhouse-admin.json b/openapi/clickhouse-admin.json index 5d1ba8464d..1a5c88f318 100644 --- a/openapi/clickhouse-admin.json +++ b/openapi/clickhouse-admin.json @@ -10,6 +10,30 @@ "version": "0.0.1" }, "paths": { + "/keeper/cluster-membership": { + "get": { + "summary": "Retrieve cluster membership information from a keeper node.", + "operationId": "keeper_cluster_membership", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClickhouseKeeperClusterMembership" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/keeper/conf": { "get": { "summary": "Retrieve configuration information from a keeper node.", @@ -199,6 +223,39 @@ } ] }, + "ClickhouseKeeperClusterMembership": { + "description": "The configuration of the clickhouse keeper raft cluster returned from a single keeper node\n\nEach keeper is asked for its known raft configuration via `clickhouse-admin` dropshot servers running in `ClickhouseKeeper` zones. state. We include the leader committed log index known to the current keeper node (whether or not it is the leader) to determine which configuration is newest.", + "type": "object", + "properties": { + "leader_committed_log_index": { + "description": "Index of the last committed log entry from the leader's perspective", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "queried_keeper": { + "description": "Keeper ID of the keeper being queried", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperId" + } + ] + }, + "raft_config": { + "description": "Keeper IDs of all keepers in the cluster", + "type": "array", + "items": { + "$ref": "#/components/schemas/KeeperId" + }, + "uniqueItems": true + } + }, + "required": [ + "leader_committed_log_index", + "queried_keeper", + "raft_config" + ] + }, "Error": { "description": "Error information from a response.", "type": "object",