diff --git a/Cargo.lock b/Cargo.lock index bcff44b32..fab79372b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -338,7 +338,7 @@ dependencies = [ "autopush_common 1.0.0", "backtrace 0.3.44 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", - "cadence 0.19.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cadence 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", "config 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "docopt 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "fernet 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -359,6 +359,7 @@ dependencies = [ "slog-term 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "validator 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "validator_derive 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -614,14 +615,6 @@ dependencies = [ "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "cadence" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "cadence" version = "0.20.0" @@ -745,14 +738,6 @@ dependencies = [ "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "crossbeam-channel" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "crossbeam-channel" version = "0.4.2" @@ -3765,7 +3750,6 @@ dependencies = [ "checksum bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" "checksum bytestring 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fc7c05fa5172da78a62d9949d662d2ac89d4cc7355d7b49adee5163f1fb3f363" "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" -"checksum cadence 0.19.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4cc8802110b3a8650896ab9ab0578b5b3057a112ccb0832b5c2ffebfccacc0db" "checksum cadence 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1c57523ff87f22fc96f07f5f379af387fdd86009242a17d805a3278540cde94a" "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" @@ -3780,7 +3764,6 @@ dependencies = [ "checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" "checksum crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" -"checksum crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa" "checksum crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" "checksum crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" "checksum crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" diff --git a/autoendpoint/Cargo.toml b/autoendpoint/Cargo.toml index b359e3739..0961b9cbb 100644 --- a/autoendpoint/Cargo.toml +++ b/autoendpoint/Cargo.toml @@ -12,7 +12,7 @@ actix-cors = "0.2.0" autopush_common = { path = "../autopush-common" } backtrace = "0.3" base64 = "0.12.1" -cadence = "0.19.1" +cadence = "0.20" config = "0.10.1" docopt = "1.1.0" fernet = "0.1.3" @@ -33,5 +33,6 @@ slog-stdlog = "4.0" slog-term = "2.5" thiserror = "1.0" url = "2.1" +uuid = { version = "0.8.1", features = ["serde", "v4"] } validator = "0.10.0" validator_derive = "0.10.0" diff --git a/autoendpoint/src/error.rs b/autoendpoint/src/error.rs index 5d4ad6f23..41e1b75c5 100644 --- a/autoendpoint/src/error.rs +++ b/autoendpoint/src/error.rs @@ -59,12 +59,21 @@ pub enum ApiErrorKind { #[error(transparent)] VapidError(#[from] VapidError), + #[error(transparent)] + Uuid(#[from] uuid::Error), + #[error("Error while validating token")] TokenHashValidation(#[from] openssl::error::ErrorStack), + #[error("Database error: {0}")] + Database(#[source] autopush_common::errors::Error), + #[error("Invalid token")] InvalidToken, + #[error("No such subscription")] + NoSubscription, + /// A specific issue with the encryption headers #[error("{0}")] InvalidEncryption(String), @@ -88,7 +97,10 @@ impl ApiErrorKind { ApiErrorKind::Validation(_) | ApiErrorKind::InvalidEncryption(_) - | ApiErrorKind::TokenHashValidation(_) => StatusCode::BAD_REQUEST, + | ApiErrorKind::TokenHashValidation(_) + | ApiErrorKind::Uuid(_) => StatusCode::BAD_REQUEST, + + ApiErrorKind::NoSubscription => StatusCode::GONE, ApiErrorKind::VapidError(_) => StatusCode::UNAUTHORIZED, @@ -96,9 +108,10 @@ impl ApiErrorKind { ApiErrorKind::PayloadTooLarge(_) => StatusCode::PAYLOAD_TOO_LARGE, - ApiErrorKind::Io(_) | ApiErrorKind::Metrics(_) | ApiErrorKind::Internal(_) => { - StatusCode::INTERNAL_SERVER_ERROR - } + ApiErrorKind::Io(_) + | ApiErrorKind::Metrics(_) + | ApiErrorKind::Database(_) + | ApiErrorKind::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/autoendpoint/src/server/extractors/mod.rs b/autoendpoint/src/server/extractors/mod.rs index 8f76dc8e6..ef4aa748e 100644 --- a/autoendpoint/src/server/extractors/mod.rs +++ b/autoendpoint/src/server/extractors/mod.rs @@ -5,3 +5,4 @@ pub mod notification; pub mod notification_headers; pub mod subscription; pub mod token_info; +pub mod user; diff --git a/autoendpoint/src/server/extractors/subscription.rs b/autoendpoint/src/server/extractors/subscription.rs index 6a2d83fda..3c978b6ca 100644 --- a/autoendpoint/src/server/extractors/subscription.rs +++ b/autoendpoint/src/server/extractors/subscription.rs @@ -1,21 +1,25 @@ use crate::error::{ApiError, ApiErrorKind, ApiResult}; use crate::server::extractors::token_info::TokenInfo; +use crate::server::extractors::user::validate_user; use crate::server::headers::crypto_key::CryptoKeyHeader; use crate::server::headers::vapid::{VapidHeader, VapidVersionData}; use crate::server::{ServerState, VapidError}; use actix_http::{Payload, PayloadStream}; use actix_web::web::Data; use actix_web::{FromRequest, HttpRequest}; +use autopush_common::db::DynamoDbUser; use cadence::{Counted, StatsdClient}; +use futures::compat::Future01CompatExt; use futures::future::LocalBoxFuture; use futures::FutureExt; use openssl::hash; use std::borrow::Cow; +use uuid::Uuid; /// Extracts subscription data from `TokenInfo` and verifies auth/crypto headers pub struct Subscription { - pub uaid: String, - pub channel_id: String, + pub user: DynamoDbUser, + pub channel_id: Uuid, pub vapid: Option, pub public_key: Option, } @@ -33,10 +37,10 @@ impl FromRequest for Subscription { let token_info = TokenInfo::extract(&req).await?; let state: Data = Data::extract(&req).await.expect("No server state found"); - let fernet = state.fernet.as_ref(); // Decrypt the token - let token = fernet + let token = state + .fernet .decrypt(&repad_base64(&token_info.token)) .map_err(|_| ApiErrorKind::InvalidToken)?; @@ -53,9 +57,20 @@ impl FromRequest for Subscription { version_1_validation(&token)?; } + // Load and validate user data + let uaid = Uuid::from_slice(&token[..16])?; + let channel_id = Uuid::from_slice(&token[16..32])?; + let user = state + .ddb + .get_user(&uaid) + .compat() + .await + .map_err(ApiErrorKind::Database)?; + validate_user(&user, &channel_id, &state).await?; + Ok(Subscription { - uaid: hex::encode(&token[..16]), - channel_id: hex::encode(&token[16..32]), + user, + channel_id, vapid, public_key, }) diff --git a/autoendpoint/src/server/extractors/user.rs b/autoendpoint/src/server/extractors/user.rs new file mode 100644 index 000000000..61135ed42 --- /dev/null +++ b/autoendpoint/src/server/extractors/user.rs @@ -0,0 +1,85 @@ +//! User validations + +use crate::error::{ApiErrorKind, ApiResult}; +use crate::server::ServerState; +use autopush_common::db::{DynamoDbUser, DynamoStorage}; +use cadence::{Counted, StatsdClient}; +use futures::compat::Future01CompatExt; +use uuid::Uuid; + +/// Valid `DynamoDbUser::router_type` values +const VALID_ROUTERS: [&str; 5] = ["webpush", "gcm", "fcm", "apns", "adm"]; + +/// Perform some validations on the user, including: +/// - Validate router type +/// - (WebPush) Check that the subscription/channel exists +/// - (WebPush) Drop user if inactive +pub async fn validate_user( + user: &DynamoDbUser, + channel_id: &Uuid, + state: &ServerState, +) -> ApiResult<()> { + if !VALID_ROUTERS.contains(&user.router_type.as_str()) { + debug!("Unknown router type, dropping user"; "user" => ?user); + drop_user(&user.uaid, &state.ddb, &state.metrics).await?; + return Err(ApiErrorKind::NoSubscription.into()); + } + + if user.router_type == "webpush" { + validate_webpush_user(user, channel_id, &state.ddb, &state.metrics).await?; + } + + Ok(()) +} + +/// Make sure the user is not inactive and the subscription channel exists +async fn validate_webpush_user( + user: &DynamoDbUser, + channel_id: &Uuid, + ddb: &DynamoStorage, + metrics: &StatsdClient, +) -> ApiResult<()> { + // Make sure the user is active (has a valid message table) + let message_table = match user.current_month.as_ref() { + Some(table) => table, + None => { + debug!("Missing `current_month` value, dropping user"; "user" => ?user); + drop_user(&user.uaid, ddb, metrics).await?; + return Err(ApiErrorKind::NoSubscription.into()); + } + }; + + if !ddb.message_table_names.contains(message_table) { + debug!("User is inactive, dropping user"; "user" => ?user); + drop_user(&user.uaid, ddb, metrics).await?; + return Err(ApiErrorKind::NoSubscription.into()); + } + + // Make sure the subscription channel exists + let channel_ids = ddb + .get_user_channels(&user.uaid, message_table) + .compat() + .await + .map_err(ApiErrorKind::Database)?; + + if !channel_ids.contains(channel_id) { + return Err(ApiErrorKind::NoSubscription.into()); + } + + Ok(()) +} + +/// Drop a user and increment associated metric +async fn drop_user(uaid: &Uuid, ddb: &DynamoStorage, metrics: &StatsdClient) -> ApiResult<()> { + metrics + .incr_with_tags("updates.drop_user") + .with_tag("errno", "102") + .send(); + + ddb.drop_uaid(uaid) + .compat() + .await + .map_err(ApiErrorKind::Database)?; + + Ok(()) +} diff --git a/autoendpoint/src/server/mod.rs b/autoendpoint/src/server/mod.rs index 177b8bcf7..420f881dd 100644 --- a/autoendpoint/src/server/mod.rs +++ b/autoendpoint/src/server/mod.rs @@ -1,18 +1,18 @@ //! Main application server -use actix_cors::Cors; -use actix_web::{ - dev, http::StatusCode, middleware::errhandlers::ErrorHandlers, web, App, HttpServer, -}; -use cadence::StatsdClient; - -use crate::error::{ApiError, ApiResult}; +use crate::error::{ApiError, ApiErrorKind, ApiResult}; use crate::metrics; use crate::server::routes::health::{ health_route, lb_heartbeat_route, status_route, version_route, }; use crate::server::routes::webpush::webpush_route; use crate::settings::Settings; +use actix_cors::Cors; +use actix_web::{ + dev, http::StatusCode, middleware::errhandlers::ErrorHandlers, web, App, HttpServer, +}; +use autopush_common::db::DynamoStorage; +use cadence::StatsdClient; use fernet::MultiFernet; use std::sync::Arc; @@ -28,6 +28,7 @@ pub struct ServerState { pub metrics: StatsdClient, pub settings: Settings, pub fernet: Arc, + pub ddb: DynamoStorage, } pub struct Server; @@ -37,10 +38,17 @@ impl Server { let metrics = metrics::metrics_from_opts(&settings)?; let bind_address = format!("{}:{}", settings.host, settings.port); let fernet = Arc::new(settings.make_fernet()); + let ddb = DynamoStorage::from_opts( + &settings.message_table_name, + &settings.router_table_name, + metrics.clone(), + ) + .map_err(ApiErrorKind::Database)?; let state = ServerState { metrics, settings, fernet, + ddb, }; let server = HttpServer::new(move || { diff --git a/autoendpoint/src/settings.rs b/autoendpoint/src/settings.rs index 7defd3362..a9986dc5c 100644 --- a/autoendpoint/src/settings.rs +++ b/autoendpoint/src/settings.rs @@ -19,6 +19,9 @@ pub struct Settings { #[cfg(any(test, feature = "db_test"))] pub database_use_test_transactions: bool, + pub router_table_name: String, + pub message_table_name: String, + pub max_data_bytes: usize, pub crypto_keys: String, pub human_logs: bool, @@ -38,6 +41,8 @@ impl Default for Settings { database_pool_max_size: None, #[cfg(any(test, feature = "db_test"))] database_use_test_transactions: false, + router_table_name: "router".to_string(), + message_table_name: "message".to_string(), max_data_bytes: 4096, crypto_keys: format!("[{}]", Fernet::generate_key()), human_logs: false, diff --git a/autopush-common/src/db/commands.rs b/autopush-common/src/db/commands.rs index 48b152089..4c465bd79 100644 --- a/autopush-common/src/db/commands.rs +++ b/autopush-common/src/db/commands.rs @@ -1,6 +1,5 @@ use std::collections::HashSet; use std::fmt::{Debug, Display}; -use std::rc::Rc; use std::result::Result as StdResult; use uuid::Uuid; @@ -11,9 +10,9 @@ use futures_backoff::retry_if; use rusoto_core::RusotoError; use rusoto_dynamodb::{ AttributeValue, BatchWriteItemError, DeleteItemError, DeleteItemInput, DeleteItemOutput, - DynamoDb, GetItemError, GetItemInput, GetItemOutput, ListTablesInput, ListTablesOutput, - PutItemError, PutItemInput, PutItemOutput, QueryError, QueryInput, UpdateItemError, - UpdateItemInput, UpdateItemOutput, + DynamoDb, DynamoDbClient, GetItemError, GetItemInput, GetItemOutput, ListTablesInput, + ListTablesOutput, PutItemError, PutItemInput, PutItemOutput, QueryError, QueryInput, + UpdateItemError, UpdateItemInput, UpdateItemOutput, }; use super::models::{DynamoDbNotification, DynamoDbUser}; @@ -63,7 +62,7 @@ fn has_connected_this_month(user: &DynamoDbUser) -> bool { /// A blocking list_tables call only called during initialization /// (prior to an any active tokio executor) pub fn list_tables_sync( - ddb: Rc>, + ddb: &DynamoDbClient, start_key: Option, ) -> Result { let input = ListTablesInput { @@ -76,8 +75,8 @@ pub fn list_tables_sync( } pub fn fetch_messages( - ddb: Rc>, - metrics: &Rc, + ddb: DynamoDbClient, + metrics: StatsdClient, table_name: &str, uaid: &Uuid, limit: u32, @@ -95,7 +94,6 @@ pub fn fetch_messages( ..Default::default() }; - let metrics = Rc::clone(metrics); retry_if(move || ddb.query(input.clone()), retryable_query_error) .chain_err(|| ErrorKind::MessageFetch) .and_then(move |output| { @@ -138,8 +136,8 @@ pub fn fetch_messages( } pub fn fetch_timestamp_messages( - ddb: Rc>, - metrics: &Rc, + ddb: DynamoDbClient, + metrics: StatsdClient, table_name: &str, uaid: &Uuid, timestamp: Option, @@ -163,7 +161,6 @@ pub fn fetch_timestamp_messages( ..Default::default() }; - let metrics = Rc::clone(metrics); retry_if(move || ddb.query(input.clone()), retryable_query_error) .chain_err(|| ErrorKind::MessageFetch) .and_then(move |output| { @@ -194,7 +191,7 @@ pub fn fetch_timestamp_messages( } pub fn drop_user( - ddb: Rc>, + ddb: DynamoDbClient, uaid: &Uuid, router_table_name: &str, ) -> impl Future { @@ -211,7 +208,7 @@ pub fn drop_user( } pub fn get_uaid( - ddb: Rc>, + ddb: DynamoDbClient, uaid: &Uuid, router_table_name: &str, ) -> impl Future { @@ -226,7 +223,7 @@ pub fn get_uaid( } pub fn register_user( - ddb: Rc>, + ddb: DynamoDbClient, user: &DynamoDbUser, router_table: &str, ) -> impl Future { @@ -268,7 +265,7 @@ pub fn register_user( } pub fn update_user_message_month( - ddb: Rc>, + ddb: DynamoDbClient, uaid: &Uuid, router_table_name: &str, message_month: &str, @@ -298,7 +295,7 @@ pub fn update_user_message_month( } pub fn all_channels( - ddb: Rc>, + ddb: DynamoDbClient, uaid: &Uuid, message_table_name: &str, ) -> impl Future, Error = Error> { @@ -328,7 +325,7 @@ pub fn all_channels( } pub fn save_channels( - ddb: Rc>, + ddb: DynamoDbClient, uaid: &Uuid, channels: HashSet, message_table_name: &str, @@ -361,7 +358,7 @@ pub fn save_channels( } pub fn unregister_channel_id( - ddb: Rc>, + ddb: DynamoDbClient, uaid: &Uuid, channel_id: &Uuid, message_table_name: &str, @@ -390,8 +387,8 @@ pub fn unregister_channel_id( #[allow(clippy::too_many_arguments)] pub fn lookup_user( - ddb: Rc>, - metrics: &Rc, + ddb: DynamoDbClient, + metrics: StatsdClient, uaid: &Uuid, connected_at: u64, router_url: &str, @@ -407,7 +404,6 @@ pub fn lookup_user( let messages_tables = message_table_names.to_vec(); let connected_at = connected_at; let router_url = router_url.to_string(); - let metrics = Rc::clone(metrics); let response = response.and_then(move |data| -> MyFuture<_> { let mut hello_response = HelloResponse { message_month: cur_month.clone(), diff --git a/autopush-common/src/db/mod.rs b/autopush-common/src/db/mod.rs index 19fa7145b..22b411dd1 100644 --- a/autopush-common/src/db/mod.rs +++ b/autopush-common/src/db/mod.rs @@ -1,6 +1,5 @@ use std::collections::HashSet; use std::env; -use std::rc::Rc; use uuid::Uuid; use cadence::StatsdClient; @@ -61,11 +60,12 @@ pub enum RegisterResponse { Error { error_msg: String, status: u32 }, } +#[derive(Clone)] pub struct DynamoStorage { - ddb: Rc>, - metrics: Rc, + ddb: DynamoDbClient, + metrics: StatsdClient, router_table_name: String, - message_table_names: Vec, + pub message_table_names: Vec, pub current_message_month: String, } @@ -75,19 +75,18 @@ impl DynamoStorage { router_table_name: &str, metrics: StatsdClient, ) -> Result { - let ddb: Box = if let Ok(endpoint) = env::var("AWS_LOCAL_DYNAMODB") { - Box::new(DynamoDbClient::new_with( + let ddb = if let Ok(endpoint) = env::var("AWS_LOCAL_DYNAMODB") { + DynamoDbClient::new_with( HttpClient::new().chain_err(|| "TLS initialization error")?, StaticProvider::new_minimal("BogusKey".to_string(), "BogusKey".to_string()), Region::Custom { name: "us-east-1".to_string(), endpoint, }, - )) + ) } else { - Box::new(DynamoDbClient::new(Region::default())) + DynamoDbClient::new(Region::default()) }; - let ddb = Rc::new(ddb); let mut message_table_names = list_message_tables(&ddb, &message_table_name) .map_err(|_| "Failed to locate message tables")?; @@ -102,7 +101,7 @@ impl DynamoStorage { Ok(Self { ddb, - metrics: Rc::new(metrics), + metrics, router_table_name: router_table_name.to_owned(), message_table_names, current_message_month, @@ -154,7 +153,7 @@ impl DynamoStorage { let response: MyFuture<(HelloResponse, Option)> = if let Some(uaid) = uaid { commands::lookup_user( self.ddb.clone(), - &self.metrics, + self.metrics.clone(), &uaid, connected_at, router_url, @@ -387,7 +386,7 @@ impl DynamoStorage { let response: MyFuture = if include_topic { Box::new(commands::fetch_messages( self.ddb.clone(), - &self.metrics, + self.metrics.clone(), table_name, uaid, 11 as u32, @@ -398,7 +397,7 @@ impl DynamoStorage { let uaid = *uaid; let table_name = table_name.to_string(); let ddb = self.ddb.clone(); - let metrics = Rc::clone(&self.metrics); + let metrics = self.metrics.clone(); response.and_then(move |resp| -> MyFuture<_> { // Return now from this future if we have messages @@ -420,7 +419,7 @@ impl DynamoStorage { if resp.messages.is_empty() || resp.timestamp.is_some() { Box::new(commands::fetch_timestamp_messages( ddb, - &metrics, + metrics, table_name.as_ref(), &uaid, timestamp, @@ -459,13 +458,27 @@ impl DynamoStorage { }); Box::new(result) } + + /// Get the set of channel IDs for a user + pub fn get_user_channels( + &self, + uaid: &Uuid, + message_table: &str, + ) -> impl Future, Error = Error> { + commands::all_channels(self.ddb.clone(), uaid, message_table).and_then(|channels| { + channels + .into_iter() + .map(|channel| channel.parse().map_err(Error::from)) + .collect::>() + }) + } } -pub fn list_message_tables(ddb: &Rc>, prefix: &str) -> Result> { +pub fn list_message_tables(ddb: &DynamoDbClient, prefix: &str) -> Result> { let mut names: Vec = Vec::new(); let mut start_key = None; loop { - let result = commands::list_tables_sync(Rc::clone(ddb), start_key)?; + let result = commands::list_tables_sync(&ddb, start_key)?; start_key = result.last_evaluated_table_name; if let Some(table_names) = result.table_names { names.extend(table_names);