diff --git a/CHANGELOG.md b/CHANGELOG.md index bf98b85ef..dd34fa48a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Bob versions changelog ## [Unreleased] #### Added - +- TLS support, TLS for grpc or rest can be enabled via cluster & node config (#303) #### Changed - Update rust edition to 2021 (#484) diff --git a/bob-apps/Cargo.toml b/bob-apps/Cargo.toml index b121afd39..a8c7e5ffd 100644 --- a/bob-apps/Cargo.toml +++ b/bob-apps/Cargo.toml @@ -38,7 +38,7 @@ serde_yaml = "0.8" stopwatch = "0.0.7" termion = "1.5" thiserror = "1.0" -tonic = { version = "0.6", features = ["prost"] } +tonic = { version = "0.6", features = ["prost", "tls"] } tower = "0.4" rand = "0.8" tower-service = "0.3" diff --git a/bob-apps/bin/bobd.rs b/bob-apps/bin/bobd.rs index 88f6e7631..7f880db2e 100755 --- a/bob-apps/bin/bobd.rs +++ b/bob-apps/bin/bobd.rs @@ -1,6 +1,6 @@ use bob::{ build_info::BuildInfo, init_counters, BobApiServer, BobServer, ClusterConfig, NodeConfig, Factory, Grinder, - VirtualMapper, BackendType, + VirtualMapper, BackendType, FactoryTlsConfig, }; use bob_access::{Authenticator, BasicAuthenticator, Credentials, StubAuthenticator, UsersMap, AuthenticationType}; use clap::{crate_version, App, Arg, ArgMatches}; @@ -123,7 +123,21 @@ async fn main() { async fn run_server(node: NodeConfig, authenticator: A, mapper: VirtualMapper, address: IpAddr, port: u16, addr: SocketAddr) { let (metrics, shared_metrics) = init_counters(&node, &addr.to_string()).await; let handle = Handle::current(); - let factory = Factory::new(node.operation_timeout(), metrics, node.name().into()); + let factory_tls_config = node.tls_config().as_ref().and_then(|tls_config| tls_config.grpc_config()) + .map(|tls_config| { + let ca_cert = std::fs::read(&tls_config.ca_cert_path).expect("can not read ca certificate from file"); + FactoryTlsConfig { + ca_cert, + tls_domain_name: tls_config.domain_name.clone(), + } + }); + let factory = Factory::new(node.operation_timeout(), metrics, node.name().into(), factory_tls_config); + + let mut server_builder = Server::builder(); + if let Some(node_tls_config) = node.tls_config().as_ref().and_then(|tls_config| tls_config.grpc_config()) { + let tls_config = node_tls_config.to_server_tls_config(); + server_builder = server_builder.tls_config(tls_config).expect("grpc tls config"); + } let bob = BobServer::new( Grinder::new(mapper, &node).await, @@ -135,10 +149,10 @@ async fn run_server(node: NodeConfig, authenticator: A, mapper bob.run_backend().await.unwrap(); create_signal_handlers(&bob).unwrap(); bob.run_periodic_tasks(factory); - bob.run_api_server(address, port); + bob.run_api_server(address, port, node.tls_config()).await; let bob_service = BobApiServer::new(bob); - Server::builder() + server_builder .tcp_nodelay(true) .add_service(bob_service) .serve(addr) diff --git a/bob-apps/bin/bobp.rs b/bob-apps/bin/bobp.rs index 8e1c54b31..dea1253cc 100644 --- a/bob-apps/bin/bobp.rs +++ b/bob-apps/bin/bobp.rs @@ -16,11 +16,12 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::thread; use std::time::{self, Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::fs; use tokio::sync::Mutex; use tokio::task::JoinHandle; use tokio::time::sleep; use tonic::metadata::{Ascii, MetadataValue}; -use tonic::transport::{Channel, Endpoint}; +use tonic::transport::{Channel, Endpoint, Certificate, ClientTlsConfig}; use tonic::{Code, Request, Status}; #[macro_use] @@ -30,6 +31,8 @@ extern crate log; struct NetConfig { port: u16, target: String, + ca_cert_path: Option, + tls_domain_name: Option, } impl NetConfig { fn get_uri(&self) -> http::Uri { @@ -45,11 +48,20 @@ impl NetConfig { Self { port: matches.value_or_default("port"), target: matches.value_or_default("host"), + ca_cert_path: matches.value_of("ca_path").map(|p| p.to_string()), + tls_domain_name: matches.value_of("domain_name").map(|n| n.to_string()), } } async fn build_client(&self) -> BobApiClient { - let endpoint = Endpoint::from(self.get_uri()).tcp_nodelay(true); + let mut endpoint = Endpoint::from(self.get_uri()).tcp_nodelay(true); + if let Some(ca_cert_path) = &self.ca_cert_path { + let cert_bin = fs::read(&ca_cert_path).expect("can not read ca certificate from file"); + let cert = Certificate::from_pem(cert_bin); + let domain_name = self.tls_domain_name.as_ref().expect("domain name required"); + let tls_config = ClientTlsConfig::new().domain_name(domain_name).ca_certificate(cert); + endpoint = endpoint.tls_config(tls_config).expect("tls config"); + } loop { match BobApiClient::connect(endpoint.clone()).await { Ok(client) => return client, @@ -1066,6 +1078,19 @@ fn get_matches() -> ArgMatches<'static> { .short("s") .default_value("1000"), ) + .arg( + Arg::with_name("ca_path") + .help("path to tls ca certificate") + .takes_value(true) + .long("ca_path") + .requires("domain_name"), + ) + .arg( + Arg::with_name("domain_name") + .help("tls domain name") + .takes_value(true) + .long("domain_name"), + ) .get_matches() } diff --git a/bob-common/Cargo.toml b/bob-common/Cargo.toml index 67b614303..f7818f638 100644 --- a/bob-common/Cargo.toml +++ b/bob-common/Cargo.toml @@ -28,7 +28,7 @@ serde = { version = "1.0", features = ["rc"] } serde_derive = "1.0" serde_yaml = "0.8" thiserror = "1.0" -tonic = { version = "0.6", features = ["prost"] } +tonic = { version = "0.6", features = ["prost", "tls"] } tower = "0.4" tower-service = "0.3" ubyte = { version = "0.10", features = ["serde"] } diff --git a/bob-common/src/bob_client.rs b/bob-common/src/bob_client.rs index 3bd6ce198..a3f576401 100644 --- a/bob-common/src/bob_client.rs +++ b/bob-common/src/bob_client.rs @@ -1,5 +1,5 @@ pub mod b_client { - use super::{ExistResult, GetResult, PingResult, PutResult}; + use super::{ExistResult, GetResult, PingResult, PutResult, FactoryTlsConfig}; use crate::{ data::{BobData, BobKey, BobMeta}, error::Error, @@ -17,7 +17,7 @@ pub mod b_client { }; use tonic::{ metadata::MetadataValue, - transport::{Channel, Endpoint}, + transport::{Certificate, Channel, ClientTlsConfig, Endpoint}, Request, Response, Status, }; @@ -41,8 +41,16 @@ pub mod b_client { operation_timeout: Duration, metrics: BobClientMetrics, local_node_name: String, + tls_config: Option<&FactoryTlsConfig>, ) -> Result { - let endpoint = Endpoint::from(node.get_uri()).tcp_nodelay(true); + let mut endpoint = Endpoint::from(node.get_uri()); + if let Some(tls_config) = tls_config { + let cert = Certificate::from_pem(&tls_config.ca_cert); + let tls_config = ClientTlsConfig::new().domain_name(&tls_config.tls_domain_name).ca_certificate(cert); + endpoint = endpoint.tls_config(tls_config).expect("client tls"); + } + endpoint = endpoint.tcp_nodelay(true); + let client = BobApiClient::connect(endpoint) .await .map_err(|e| e.to_string())?; @@ -188,7 +196,7 @@ pub mod b_client { mock! { pub BobClient { - pub async fn create(node: Node, operation_timeout: Duration, metrics: BobClientMetrics, local_node_name: String) -> Result; + pub async fn create<'a>(node: Node, operation_timeout: Duration, metrics: BobClientMetrics, local_node_name: String, tls_config: Option<&'a FactoryTlsConfig>) -> Result; pub async fn put(&self, key: BobKey, d: BobData, options: PutOptions) -> PutResult; pub async fn get(&self, key: BobKey, options: GetOptions) -> GetResult; pub async fn ping(&self) -> PingResult; @@ -240,12 +248,19 @@ pub type PingResult = Result, NodeOutput>; pub type ExistResult = Result>, NodeOutput>; +#[derive(Clone)] +pub struct FactoryTlsConfig { + pub tls_domain_name: String, + pub ca_cert: Vec, +} + /// Bob metrics factory #[derive(Clone)] pub struct Factory { operation_timeout: Duration, metrics: Arc, local_node_name: String, + tls_config: Option, } impl Factory { @@ -255,16 +270,18 @@ impl Factory { operation_timeout: Duration, metrics: Arc, local_node_name: String, + tls_config: Option, ) -> Self { Factory { operation_timeout, metrics, local_node_name, + tls_config, } } pub async fn produce(&self, node: Node) -> Result { let metrics = self.metrics.clone().get_metrics(&node.counter_display()); - BobClient::create(node, self.operation_timeout, metrics, self.local_node_name.clone()).await + BobClient::create(node, self.operation_timeout, metrics, self.local_node_name.clone(), self.tls_config.as_ref()).await } } diff --git a/bob-common/src/configs/node.rs b/bob-common/src/configs/node.rs index ddcd3db78..bccc3f365 100755 --- a/bob-common/src/configs/node.rs +++ b/bob-common/src/configs/node.rs @@ -15,8 +15,9 @@ use std::{ time::Duration, }; use std::{net::IpAddr, sync::atomic::Ordering}; -use std::{net::Ipv4Addr, sync::Arc}; +use std::{net::Ipv4Addr, sync::Arc, fs}; use tokio::time::sleep; +use tonic::transport::{ServerTlsConfig, Identity}; use ubyte::ByteUnit; @@ -454,6 +455,46 @@ impl Validatable for Pearl { } } +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct TLSConfig { + pub ca_cert_path: String, + pub domain_name: String, + pub rest: Option, + pub grpc: Option, + pub cert_path: Option, + pub pkey_path: Option, +} + +impl TLSConfig { + pub fn grpc_config(&self) -> Option<&Self> { + self.grpc.and_then(|grpc| + if grpc { + Some(self) + } else { + None + }) + } + + pub fn rest_config(&self) -> Option<&Self> { + self.rest.and_then(|rest| + if rest { + Some(self) + } else { + None + }) + } + + pub fn to_server_tls_config(&self) -> ServerTlsConfig { + let cert_path = self.cert_path.as_ref().expect("no certificate path specified"); + let cert_bin = fs::read(cert_path).expect("can not read tls certificate from file"); + let pkey_path = self.pkey_path.as_ref().expect("no private key path specified"); + let key_bin = fs::read(pkey_path).expect("can not read tls private key from file"); + let identity = Identity::from_pem(cert_bin.clone(), key_bin); + + ServerTlsConfig::new().identity(identity) + } +} + #[derive(Debug, PartialEq, Serialize, Deserialize, Clone, Copy)] pub enum BackendType { InMemory = 0, @@ -477,6 +518,7 @@ pub struct Node { backend_type: String, pearl: Option, metrics: Option, + tls: Option, #[serde(skip)] bind_ref: Arc>, @@ -535,6 +577,10 @@ impl NodeConfig { self.metrics.as_ref().expect("metrics config") } + pub fn tls_config(&self) -> &Option { + &self.tls + } + /// Get log config file path. pub fn log_config(&self) -> &str { &self.log_config @@ -808,6 +854,7 @@ pub mod tests { backend_type: "in_memory".to_string(), pearl: None, metrics: None, + tls: None, bind_ref: Arc::default(), disks_ref: Arc::default(), cleanup_interval: "1d".to_string(), diff --git a/bob/Cargo.toml b/bob/Cargo.toml index 1848b83f6..fce8131e2 100644 --- a/bob/Cargo.toml +++ b/bob/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" anyhow = "1.0" async-trait = "0.1" axum = "0.4" +axum-server = { version = "0.3.3", features = ["tls-rustls"] } bitflags = "1.3" bob-access = { path = "../bob-access" } bob-backend = { path = "../bob-backend" } diff --git a/bob/src/api/mod.rs b/bob/src/api/mod.rs index 511205d83..b2667372e 100644 --- a/bob/src/api/mod.rs +++ b/bob/src/api/mod.rs @@ -6,6 +6,7 @@ use axum::{ routing::{delete, get, post, MethodRouter}, Json, Router, Server, }; +use axum_server::{bind_rustls, tls_rustls::{RustlsConfig, RustlsAcceptor}, Server as AxumServer}; pub(crate) use bob_access::Error as AuthError; use bob_access::{Authenticator, CredentialsHolder}; @@ -14,6 +15,7 @@ use bob_common::{ data::{BobData, BobKey, BobMeta, BobOptions, VDisk as DataVDisk, BOB_KEY_SIZE}, error::Error as BobError, node::Disk as NodeDisk, + configs::node::TLSConfig, }; use bytes::Bytes; use futures::{future::BoxFuture, stream::FuturesUnordered, FutureExt, StreamExt}; @@ -143,16 +145,35 @@ pub(crate) struct SpaceInfo { occupied_disk_space_by_disk: HashMap } -pub(crate) fn spawn(bob: BobServer, address: IpAddr, port: u16) +async fn tls_server(tls_config: &TLSConfig, addr: SocketAddr) -> AxumServer { + if let (Some(cert_path), Some(pkey_path)) = (&tls_config.cert_path, &tls_config.pkey_path) { + let config = RustlsConfig::from_pem_file( + cert_path, + pkey_path, + ).await.expect("can not create tls config from pem file"); + bind_rustls(addr, config) + } else { + error!("rest tls enabled, but certificate or private key not specified"); + panic!("rest tls enabled, but certificate or private key not specified"); + } +} + +pub(crate) async fn spawn(bob: BobServer, address: IpAddr, port: u16, tls_config: &Option) where A: Authenticator, { let socket_addr = SocketAddr::new(address, port); let router = router::().layer(Extension(bob)); - let task = Server::bind(&socket_addr).serve(router.into_make_service()); - - tokio::spawn(task); + + if let Some(tls_config) = tls_config.as_ref().and_then(|tls_config| tls_config.rest_config()) { + let tls_server = tls_server(&tls_config, socket_addr).await; + let task = tls_server.serve(router.into_make_service()); + tokio::spawn(task); + } else { + let task = Server::bind(&socket_addr).serve(router.into_make_service()); + tokio::spawn(task); + } info!("API server started, listening: {}", socket_addr); } diff --git a/bob/src/lib.rs b/bob/src/lib.rs index f45718684..4b422121e 100644 --- a/bob/src/lib.rs +++ b/bob/src/lib.rs @@ -28,7 +28,7 @@ pub mod server; pub use crate::{grinder::Grinder, server::Server as BobServer}; pub use bob_backend::pearl::Key as PearlKey; pub use bob_common::{ - bob_client::Factory, + bob_client::{Factory, FactoryTlsConfig}, configs::cluster::{ Cluster as ClusterConfig, Node as ClusterNodeConfig, Rack as ClusterRackConfig, Replica as ReplicaConfig, VDisk as VDiskConfig, diff --git a/bob/src/server.rs b/bob/src/server.rs index f42ecd974..3acf1a905 100644 --- a/bob/src/server.rs +++ b/bob/src/server.rs @@ -6,7 +6,7 @@ use tokio::{runtime::Handle, task::block_in_place}; use crate::prelude::*; use super::grinder::Grinder; -use bob_common::metrics::SharedMetricsSnapshot; +use bob_common::{metrics::SharedMetricsSnapshot, configs::node::TLSConfig}; /// Struct contains `Grinder` and receives incomming GRPC requests #[derive(Clone, Debug)] @@ -50,8 +50,8 @@ where } /// Call to run HTTP API server, not required for normal functioning - pub fn run_api_server(&self, address: IpAddr, port: u16) { - crate::api::spawn(self.clone(), address, port); + pub async fn run_api_server(&self, address: IpAddr, port: u16, tls_config: &Option) { + crate::api::spawn(self.clone(), address, port, tls_config).await; } /// Start backend component, required before starting bob service diff --git a/config-examples/cluster.yaml b/config-examples/cluster.yaml index b3b250a36..04074e803 100644 --- a/config-examples/cluster.yaml +++ b/config-examples/cluster.yaml @@ -11,7 +11,7 @@ nodes: # [str] node name - name: local_node # [ip:port] node address - address: 127.0.0.1:20000 + address: 127.0.0.1:20000 # [list] of physical node disks disks: # [str] name of disk, the node uses it to determine which vdisks belong to it diff --git a/config-examples/node.yaml b/config-examples/node.yaml index 40481361a..b3bee371c 100644 --- a/config-examples/node.yaml +++ b/config-examples/node.yaml @@ -62,6 +62,21 @@ authentication_type: Basic # [size] memory limit for all indexes. Unlimited if not specified index_memory_limit: 8 GiB +# tls parameters +tls: + # [file] ca certificate to verify other tls nodes + ca_cert_path: my_ca.pem + # [file] node certificate + cert_path: server.pem + # [file] node private key + pkey_path: server.key + # enable tls for rest api + rest: false + # enable tls for grpc + grpc: false + # specify tls domain name + domain_name: bob + # used only for 'backend_type: pearl' pearl: # optional, default = false, enables linux AIO