diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index ea80a6d34b6..9b9bde79c80 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -1169,29 +1169,55 @@ impl ZoneBuilderFactory { /// Created by [ZoneBuilderFactory]. #[derive(Default)] pub struct ZoneBuilder<'a> { + /// Logger to which status messages are written during zone installation. log: Option, + /// Allocates the NIC used for control plane communication. underlay_vnic_allocator: Option<&'a VnicAllocator>, + /// Filesystem path at which the installed zone will reside. zone_root_path: Option<&'a Utf8Path>, + /// The directories that will be searched for the image tarball for the + /// provided zone type ([`Self::with_zone_type`]). zone_image_paths: Option<&'a [Utf8PathBuf]>, + /// The name of the type of zone being created (e.g. "propolis-server") zone_type: Option<&'a str>, - unique_name: Option, // actually optional + /// Unique ID of the instance of the zone being created. (optional) + // *actually* optional (in contrast to other fields that are `Option` for + // builder purposes - that is, skipping this field in the builder will + // still result in an `Ok(InstalledZone)` from `.install()`, rather than + // an `Err(InstallZoneError::IncompleteBuilder)`. + unique_name: Option, + /// ZFS datasets to be accessed from within the zone. datasets: Option<&'a [zone::Dataset]>, + /// Filesystems to mount within the zone. filesystems: Option<&'a [zone::Fs]>, + /// Additional network device names to add to the zone. data_links: Option<&'a [String]>, + /// Device nodes to pass through to the zone. devices: Option<&'a [zone::Device]>, + /// OPTE devices for the guest network interfaces. opte_ports: Option>, - bootstrap_vnic: Option, // actually optional + /// NIC to use for creating a bootstrap address on the switch zone. + // actually optional (as above) + bootstrap_vnic: Option, + /// Physical NICs possibly provisioned to the zone. links: Option>, + /// The maximum set of privileges any process in this zone can obtain. limit_priv: Option>, + /// For unit tests only: if `Some`, then no actual zones will be installed + /// by this builder, and minimal facsimiles of them will be placed in + /// temporary directories according to the contents of the provided + /// `FakeZoneBuilderConfig`. fake_cfg: Option, } impl<'a> ZoneBuilder<'a> { + /// Logger to which status messages are written during zone installation. pub fn with_log(mut self, log: Logger) -> Self { self.log = Some(log); self } + /// Allocates the NIC used for control plane communication. pub fn with_underlay_vnic_allocator( mut self, vnic_allocator: &'a VnicAllocator, @@ -1200,11 +1226,14 @@ impl<'a> ZoneBuilder<'a> { self } + /// Filesystem path at which the installed zone will reside. pub fn with_zone_root_path(mut self, root_path: &'a Utf8Path) -> Self { self.zone_root_path = Some(root_path); self } + /// The directories that will be searched for the image tarball for the + /// provided zone type ([`Self::with_zone_type`]). pub fn with_zone_image_paths( mut self, image_paths: &'a [Utf8PathBuf], @@ -1213,56 +1242,68 @@ impl<'a> ZoneBuilder<'a> { self } + /// The name of the type of zone being created (e.g. "propolis-server") pub fn with_zone_type(mut self, zone_type: &'a str) -> Self { self.zone_type = Some(zone_type); self } + /// Unique ID of the instance of the zone being created. (optional) pub fn with_unique_name(mut self, uuid: Uuid) -> Self { self.unique_name = Some(uuid); self } + /// ZFS datasets to be accessed from within the zone. pub fn with_datasets(mut self, datasets: &'a [zone::Dataset]) -> Self { self.datasets = Some(datasets); self } + /// Filesystems to mount within the zone. pub fn with_filesystems(mut self, filesystems: &'a [zone::Fs]) -> Self { self.filesystems = Some(filesystems); self } + /// Additional network device names to add to the zone. pub fn with_data_links(mut self, links: &'a [String]) -> Self { self.data_links = Some(links); self } + /// Device nodes to pass through to the zone. pub fn with_devices(mut self, devices: &'a [zone::Device]) -> Self { self.devices = Some(devices); self } + /// OPTE devices for the guest network interfaces. pub fn with_opte_ports(mut self, ports: Vec<(Port, PortTicket)>) -> Self { self.opte_ports = Some(ports); self } + /// NIC to use for creating a bootstrap address on the switch zone. + /// (optional) pub fn with_bootstrap_vnic(mut self, vnic: Link) -> Self { self.bootstrap_vnic = Some(vnic); self } + /// Physical NICs possibly provisioned to the zone. pub fn with_links(mut self, links: Vec) -> Self { self.links = Some(links); self } + /// The maximum set of privileges any process in this zone can obtain. pub fn with_limit_priv(mut self, limit_priv: Vec) -> Self { self.limit_priv = Some(limit_priv); self } + // (used in unit tests) fn fake_install(self) -> Result { let zone = self .zone_type @@ -1300,6 +1341,9 @@ impl<'a> ZoneBuilder<'a> { .ok_or(InstallZoneError::IncompleteBuilder) } + /// Create the zone with the provided parameters. + /// Returns `Err(InstallZoneError::IncompleteBuilder)` if a necessary + /// parameter was not provided. pub async fn install(self) -> Result { if self.fake_cfg.is_some() { return self.fake_install(); diff --git a/sled-agent/src/fakes/nexus.rs b/sled-agent/src/fakes/nexus.rs index 5ba4b6c501b..1497e5cfef8 100644 --- a/sled-agent/src/fakes/nexus.rs +++ b/sled-agent/src/fakes/nexus.rs @@ -8,13 +8,18 @@ //! to operate correctly. use dropshot::{ - endpoint, ApiDescription, FreeformBody, HttpError, HttpResponseOk, Path, - RequestContext, + endpoint, ApiDescription, FreeformBody, HttpError, HttpResponseOk, + HttpResponseUpdatedNoContent, Path, RequestContext, TypedBody, }; use hyper::Body; use internal_dns::ServiceName; use omicron_common::api::external::Error; -use omicron_common::api::internal::nexus::UpdateArtifactId; +use omicron_common::api::internal::nexus::{ + SledInstanceState, UpdateArtifactId, +}; +use schemars::JsonSchema; +use serde::Deserialize; +use uuid::Uuid; /// Implements a fake Nexus. /// @@ -28,6 +33,13 @@ pub trait FakeNexusServer: Send + Sync { ) -> Result, Error> { Err(Error::internal_error("Not implemented")) } + fn cpapi_instances_put( + &self, + _instance_id: Uuid, + _new_runtime_state: SledInstanceState, + ) -> Result<(), Error> { + Err(Error::internal_error("Not implemented")) + } } /// Describes the server context type. @@ -52,9 +64,32 @@ async fn cpapi_artifact_download( )) } +#[derive(Deserialize, JsonSchema)] +struct InstancePathParam { + instance_id: Uuid, +} + +#[endpoint { + method = PUT, + path = "/instances/{instance_id}", +}] +async fn cpapi_instances_put( + request_context: RequestContext, + path_params: Path, + new_runtime_state: TypedBody, +) -> Result { + let context = request_context.context(); + context.cpapi_instances_put( + path_params.into_inner().instance_id, + new_runtime_state.into_inner(), + )?; + Ok(HttpResponseUpdatedNoContent()) +} + fn api() -> ApiDescription { let mut api = ApiDescription::new(); api.register(cpapi_artifact_download).unwrap(); + api.register(cpapi_instances_put).unwrap(); api } diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index c37f0ffde68..72fe3176ba8 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -1090,3 +1090,476 @@ impl Instance { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::fakes::nexus::{FakeNexusServer, ServerContext}; + use crate::nexus::NexusClient; + use crate::zone_bundle::CleanupContext; + use camino_tempfile::Utf8TempDir; + use dns_server::dns_server::ServerHandle as DnsServerHandle; + use dropshot::test_util::LogContext; + use dropshot::{HandlerTaskMode, HttpServer}; + use illumos_utils::dladm::MockDladm; + use illumos_utils::dladm::__mock_MockDladm::__create_vnic::Context as MockDladmCreateVnicContext; + use illumos_utils::dladm::__mock_MockDladm::__delete_vnic::Context as MockDladmDeleteVnicContext; + use illumos_utils::opte::params::DhcpConfig; + use illumos_utils::svc::__wait_for_service::Context as MockWaitForServiceContext; + use illumos_utils::zone::MockZones; + use illumos_utils::zone::__mock_MockZones::__boot::Context as MockZonesBootContext; + use illumos_utils::zone::__mock_MockZones::__id::Context as MockZonesIdContext; + use illumos_utils::zpool::ZpoolName; + use internal_dns::resolver::Resolver; + use internal_dns::ServiceName; + use omicron_common::api::external::{ + ByteCount, Generation, InstanceCpuCount, InstanceState, + }; + use omicron_common::api::internal::nexus::InstanceProperties; + use sled_storage::disk::{RawDisk, SyntheticDisk}; + use sled_storage::manager::FakeStorageManager; + use std::net::Ipv6Addr; + use tokio::sync::watch::Receiver; + use tokio::time::timeout; + + const TIMEOUT_DURATION: tokio::time::Duration = + tokio::time::Duration::from_secs(3); + + struct NexusServer { + observed_runtime_state: + tokio::sync::watch::Sender>, + } + impl FakeNexusServer for NexusServer { + fn cpapi_instances_put( + &self, + _instance_id: Uuid, + new_runtime_state: SledInstanceState, + ) -> Result<(), omicron_common::api::external::Error> { + self.observed_runtime_state.send(Some(new_runtime_state)) + .map_err(|_| omicron_common::api::external::Error::internal_error("couldn't send updated SledInstanceState to test driver")) + } + } + + fn fake_nexus_server( + logctx: &LogContext, + ) -> ( + NexusClient, + HttpServer, + Receiver>, + ) { + let (state_tx, state_rx) = tokio::sync::watch::channel(None); + + let nexus_server = crate::fakes::nexus::start_test_server( + logctx.log.new(o!("component" => "FakeNexusServer")), + Box::new(NexusServer { observed_runtime_state: state_tx }), + ); + let nexus_client = NexusClient::new( + &format!("http://{}", nexus_server.local_addr()), + logctx.log.new(o!("component" => "NexusClient")), + ); + + (nexus_client, nexus_server, state_rx) + } + + fn mock_vnic_contexts( + ) -> (MockDladmCreateVnicContext, MockDladmDeleteVnicContext) { + let create_vnic_ctx = MockDladm::create_vnic_context(); + let delete_vnic_ctx = MockDladm::delete_vnic_context(); + create_vnic_ctx.expect().return_once( + |physical_link: &Etherstub, _, _, _, _| { + assert_eq!(&physical_link.0, "mystub"); + Ok(()) + }, + ); + delete_vnic_ctx.expect().returning(|_| Ok(())); + (create_vnic_ctx, delete_vnic_ctx) + } + + // InstanceManager::ensure_state calls Instance::put_state(Running), + // which calls Instance::propolis_ensure, + // which spawns Instance::monitor_state_task, + // which calls cpapi_instances_put + // and calls Instance::setup_propolis_locked, + // which creates the zone (which isn't real in these tests, of course) + fn mock_zone_contexts( + ) -> (MockZonesBootContext, MockWaitForServiceContext, MockZonesIdContext) + { + let boot_ctx = MockZones::boot_context(); + boot_ctx.expect().return_once(|_| Ok(())); + let wait_ctx = illumos_utils::svc::wait_for_service_context(); + wait_ctx.expect().times(..).returning(|_, _, _| Ok(())); + let zone_id_ctx = MockZones::id_context(); + zone_id_ctx.expect().times(..).returning(|_| Ok(Some(1))); + (boot_ctx, wait_ctx, zone_id_ctx) + } + + async fn dns_server( + logctx: &LogContext, + nexus_server: &HttpServer, + ) -> (DnsServerHandle, Arc, Utf8TempDir) { + let storage_path = + Utf8TempDir::new().expect("Failed to create temporary directory"); + let config_store = dns_server::storage::Config { + keep_old_generations: 3, + storage_path: storage_path.path().to_owned(), + }; + + let (dns_server, dns_dropshot) = dns_server::start_servers( + logctx.log.new(o!("component" => "DnsServer")), + dns_server::storage::Store::new( + logctx.log.new(o!("component" => "DnsStore")), + &config_store, + ) + .unwrap(), + &dns_server::dns_server::Config { + bind_address: "[::1]:0".parse().unwrap(), + }, + &dropshot::ConfigDropshot { + bind_address: "[::1]:0".parse().unwrap(), + request_body_max_bytes: 8 * 1024, + default_handler_task_mode: HandlerTaskMode::Detached, + }, + ) + .await + .expect("starting DNS server"); + + let dns_dropshot_client = dns_service_client::Client::new( + &format!("http://{}", dns_dropshot.local_addr()), + logctx.log.new(o!("component" => "DnsDropshotClient")), + ); + let mut dns_config = internal_dns::DnsConfigBuilder::new(); + let IpAddr::V6(nexus_ip_addr) = nexus_server.local_addr().ip() else { + panic!("IPv6 address required for nexus_server") + }; + let zone = dns_config.host_zone(Uuid::new_v4(), nexus_ip_addr).unwrap(); + dns_config + .service_backend_zone( + ServiceName::Nexus, + &zone, + nexus_server.local_addr().port(), + ) + .unwrap(); + let dns_config = dns_config.build(); + dns_dropshot_client.dns_config_put(&dns_config).await.unwrap(); + + let resolver = Arc::new( + Resolver::new_from_addrs( + logctx.log.new(o!("component" => "Resolver")), + &[dns_server.local_address()], + ) + .unwrap(), + ); + (dns_server, resolver, storage_path) + } + + // note the "mock" here is different from the vnic/zone contexts above. + // this is actually running code for a dropshot server from propolis. + // (might we want a locally-defined fake whose behavior we can control + // more directly from the test driver?) + // TODO: factor out, this is also in sled-agent-sim. + fn propolis_mock_server( + log: &Logger, + ) -> (HttpServer>, PropolisClient) { + let propolis_bind_address = + SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 0); // allocate port + let dropshot_config = dropshot::ConfigDropshot { + bind_address: propolis_bind_address, + ..Default::default() + }; + let propolis_log = log.new(o!("component" => "propolis-server-mock")); + let private = + Arc::new(propolis_mock_server::Context::new(propolis_log)); + info!(log, "Starting mock propolis-server..."); + let dropshot_log = log.new(o!("component" => "dropshot")); + let mock_api = propolis_mock_server::api(); + + let srv = dropshot::HttpServerStarter::new( + &dropshot_config, + mock_api, + private, + &dropshot_log, + ) + .expect("couldn't create mock propolis-server") + .start(); + + let client = propolis_client::Client::new(&format!( + "http://{}", + srv.local_addr() + )); + + (srv, client) + } + + // make a FakeStorageManager with a "U2" upserted + async fn fake_storage_manager_with_u2() -> StorageHandle { + let (storage_manager, storage_handle) = FakeStorageManager::new(); + tokio::spawn(storage_manager.run()); + let external_zpool_name = ZpoolName::new_external(Uuid::new_v4()); + let external_disk: RawDisk = + SyntheticDisk::new(external_zpool_name).into(); + storage_handle.upsert_disk(external_disk).await; + storage_handle + } + + async fn instance_struct( + logctx: &LogContext, + propolis_addr: SocketAddr, + nexus_client_with_resolver: NexusClientWithResolver, + storage_handle: StorageHandle, + ) -> Instance { + let id = Uuid::new_v4(); + let propolis_id = Uuid::new_v4(); + let ticket = InstanceTicket::new_without_manager_for_test(id); + let hardware = InstanceHardware { + properties: InstanceProperties { + ncpus: InstanceCpuCount(1), + memory: ByteCount::from_gibibytes_u32(1), + hostname: "bert".to_string(), + }, + nics: vec![], + source_nat: SourceNatConfig { + ip: IpAddr::V6(Ipv6Addr::UNSPECIFIED), + first_port: 0, + last_port: 0, + }, + external_ips: vec![], + firewall_rules: vec![], + dhcp_config: DhcpConfig { + dns_servers: vec![], + host_domain: None, + search_domains: vec![], + }, + disks: vec![], + cloud_init_bytes: None, + }; + + let initial_state = InstanceInitialState { + hardware, + instance_runtime: InstanceRuntimeState { + propolis_id: Some(propolis_id), + dst_propolis_id: None, + migration_id: None, + gen: Generation::new(), + time_updated: Default::default(), + }, + vmm_runtime: VmmRuntimeState { + state: InstanceState::Creating, + gen: Generation::new(), + time_updated: Default::default(), + }, + propolis_addr, + }; + + let vnic_allocator = + VnicAllocator::new("Foo", Etherstub("mystub".to_string())); + let port_manager = PortManager::new( + logctx.log.new(o!("component" => "PortManager")), + Ipv6Addr::new(0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01), + ); + + let cleanup_context = CleanupContext::default(); + let zone_bundler = ZoneBundler::new( + logctx.log.new(o!("component" => "ZoneBundler")), + storage_handle.clone(), + cleanup_context, + ); + + let services = InstanceManagerServices { + nexus_client: nexus_client_with_resolver, + vnic_allocator, + port_manager, + storage: storage_handle, + zone_bundler, + zone_builder_factory: ZoneBuilderFactory::fake(), + }; + + Instance::new( + logctx.log.new(o!("component" => "Instance")), + id, + propolis_id, + ticket, + initial_state, + services, + ) + .unwrap() + } + + #[tokio::test] + async fn test_instance_create_events_normal() { + let logctx = omicron_test_utils::dev::test_setup_log( + "test_instance_create_events_normal", + ); + + let (propolis_server, _propolis_client) = + propolis_mock_server(&logctx.log); + let propolis_addr = propolis_server.local_addr(); + + // automock'd things used during this test + let _mock_vnic_contexts = mock_vnic_contexts(); + let _mock_zone_contexts = mock_zone_contexts(); + + let (nexus_client, nexus_server, mut state_rx) = + fake_nexus_server(&logctx); + + let (_dns_server, resolver, _dns_config_dir) = + timeout(TIMEOUT_DURATION, dns_server(&logctx, &nexus_server)) + .await + .expect("timed out making DNS server and Resolver"); + + let nexus_client_with_resolver = + NexusClientWithResolver::new_with_client(nexus_client, resolver); + + let storage_handle = fake_storage_manager_with_u2().await; + + let inst = timeout( + TIMEOUT_DURATION, + instance_struct( + &logctx, + propolis_addr, + nexus_client_with_resolver, + storage_handle, + ), + ) + .await + .expect("timed out creating Instance struct"); + + timeout( + TIMEOUT_DURATION, + inst.put_state(InstanceStateRequested::Running), + ) + .await + .expect("timed out waiting for Instance::put_state") + .unwrap(); + + timeout( + TIMEOUT_DURATION, + state_rx.wait_for(|maybe_state| { + maybe_state + .as_ref() + .map(|sled_inst_state| { + sled_inst_state.vmm_state.state + == InstanceState::Running + }) + .unwrap_or(false) + }), + ) + .await + .expect("timed out waiting for InstanceState::Running in FakeNexus") + .unwrap(); + + logctx.cleanup_successful(); + } + + // tests around dropshot request timeouts during the blocking propolis setup + #[tokio::test] + async fn test_instance_create_timeout_while_starting_propolis() { + let logctx = omicron_test_utils::dev::test_setup_log( + "test_instance_create_timeout_while_starting_propolis", + ); + + // automock'd things used during this test + let _mock_vnic_contexts = mock_vnic_contexts(); + let _mock_zone_contexts = mock_zone_contexts(); + + let (nexus_client, nexus_server, state_rx) = fake_nexus_server(&logctx); + + let (_dns_server, resolver, _dns_config_dir) = + timeout(TIMEOUT_DURATION, dns_server(&logctx, &nexus_server)) + .await + .expect("timed out making DNS server and Resolver"); + + let nexus_client_with_resolver = + NexusClientWithResolver::new_with_client(nexus_client, resolver); + + let storage_handle = fake_storage_manager_with_u2().await; + + let inst = timeout( + TIMEOUT_DURATION, + instance_struct( + &logctx, + // we want to test propolis not ever coming up + SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 1, 0, 0)), + nexus_client_with_resolver, + storage_handle, + ), + ) + .await + .expect("timed out creating Instance struct"); + + timeout(TIMEOUT_DURATION, inst.put_state(InstanceStateRequested::Running)) + .await + .expect_err("*should've* timed out waiting for Instance::put_state, but didn't?"); + + if let Some(SledInstanceState { + vmm_state: VmmRuntimeState { state: InstanceState::Running, .. }, + .. + }) = state_rx.borrow().to_owned() + { + panic!("Nexus's InstanceState should never have reached running if zone creation timed out"); + } + + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_instance_create_timeout_while_creating_zone() { + let logctx = omicron_test_utils::dev::test_setup_log( + "test_instance_create_timeout_while_creating_zone", + ); + + // automock'd things used during this test + let _mock_vnic_contexts = mock_vnic_contexts(); + + // time out while booting zone, on purpose! + let boot_ctx = MockZones::boot_context(); + boot_ctx.expect().return_once(|_| { + std::thread::sleep(TIMEOUT_DURATION * 2); + Ok(()) + }); + let wait_ctx = illumos_utils::svc::wait_for_service_context(); + wait_ctx.expect().times(..).returning(|_, _, _| Ok(())); + let zone_id_ctx = MockZones::id_context(); + zone_id_ctx.expect().times(..).returning(|_| Ok(Some(1))); + let halt_rm_ctx = MockZones::halt_and_remove_logged_context(); + halt_rm_ctx.expect().times(..).returning(|_, _| Ok(())); + + let (nexus_client, nexus_server, state_rx) = fake_nexus_server(&logctx); + + let (_dns_server, resolver, _dns_config_dir) = + timeout(TIMEOUT_DURATION, dns_server(&logctx, &nexus_server)) + .await + .expect("timed out making DNS server and Resolver"); + + let nexus_client_with_resolver = + NexusClientWithResolver::new_with_client(nexus_client, resolver); + + let storage_handle = fake_storage_manager_with_u2().await; + + let inst = timeout( + TIMEOUT_DURATION, + instance_struct( + &logctx, + // isn't running because the "zone" never "boots" + SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 1, 0, 0)), + nexus_client_with_resolver, + storage_handle, + ), + ) + .await + .expect("timed out creating Instance struct"); + + timeout(TIMEOUT_DURATION, inst.put_state(InstanceStateRequested::Running)) + .await + .expect_err("*should've* timed out waiting for Instance::put_state, but didn't?"); + + if let Some(SledInstanceState { + vmm_state: VmmRuntimeState { state: InstanceState::Running, .. }, + .. + }) = state_rx.borrow().to_owned() + { + panic!("Nexus's InstanceState should never have reached running if zone creation timed out"); + } + + logctx.cleanup_successful(); + } +} diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index c1b7e402a49..1d9db9d4127 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -449,6 +449,11 @@ impl InstanceTicket { InstanceTicket { id, inner: Some(inner) } } + #[cfg(test)] + pub(crate) fn new_without_manager_for_test(id: Uuid) -> Self { + Self { id, inner: None } + } + /// Idempotently removes this instance from the tracked set of /// instances. This acts as an "upcall" for instances to remove /// themselves after stopping. diff --git a/sled-agent/src/nexus.rs b/sled-agent/src/nexus.rs index cc715f4010b..7fdcc7176af 100644 --- a/sled-agent/src/nexus.rs +++ b/sled-agent/src/nexus.rs @@ -56,6 +56,16 @@ impl NexusClientWithResolver { } } + // for when we have a NexusClient constructed from a FakeNexusServer + // (no need to expose this function outside of tests) + #[cfg(test)] + pub(crate) fn new_with_client( + client: NexusClient, + resolver: Arc, + ) -> Self { + Self { client, resolver } + } + /// Access the progenitor-based Nexus Client. pub fn client(&self) -> &NexusClient { &self.client