feat: support dual mode data storage

.circleci/config.yml

@@ -25,7 +25,7 @@ jobs:
   audit:
     docker:
       # NOTE: update version for all # RUST_VER
-      - image: rust:1.72
+      - image: rust:1.73
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
@@ -105,7 +105,7 @@ jobs:
             apt install build-essential curl libstdc++6 libstdc++-10-dev libssl-dev pkg-config -y
             apt install cmake -y
             # RUST_VER
-            curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.72 -y
+            curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.73 -y
             export PATH=$PATH:$HOME/.cargo/bin
             echo 'export PATH=$PATH:$HOME/.cargo/bin' >> $BASH_ENV
             rustc --version

Cargo.toml

@@ -27,6 +27,7 @@ edition = "2021"
 # ideally, this would contain any crates that are shared between crates.
 # there are some lingering code interdependencies that prevent that, but it should
 # remain a goal for the overall refactor.
+
 actix = "0.13"
 actix-cors = "0.6"
 actix-http = "3.2"
@@ -77,7 +78,9 @@ rusoto_dynamodb = { version = "0.47", default-features = false, features = [
 sentry = { version = "0.31", features = [
   "debug-logs",
 ] } # Using debug-logs avoids https://github.com/getsentry/sentry-rust/issues/237
-sentry-core = { version = "0.31" }
+sentry-core = { version = "0.31", default-features = false, features = [
+  "client",
+] }
 sentry-actix = "0.31"
 sentry-backtrace = "0.31"
 serde = "1.0"
@@ -112,7 +115,10 @@ autoconnect_settings = { path = "./autoconnect/autoconnect-settings" }
 autoconnect_web = { path = "./autoconnect/autoconnect-web" }
 autoconnect_ws = { path = "./autoconnect/autoconnect-ws" }
 autoconnect_ws_clientsm = { path = "./autoconnect/autoconnect-ws/autoconnect-ws-clientsm" }
-autopush_common = { path = "./autopush-common", features = ["bigtable"] }
+autopush_common = { path = "./autopush-common", features = [
+  "dynamodb",
+  "bigtable",
+] }
 
 [profile.release]
 debug = 1

Dockerfile

@@ -1,5 +1,5 @@
 # NOTE: Ensure builder's Rust version matches CI's in .circleci/config.yml
-FROM rust:1.72-buster as builder
+FROM rust:1.73-buster as builder
 ARG CRATE
 
 ADD . /app

autoendpoint/Cargo.toml

@@ -44,9 +44,12 @@ tokio.workspace = true
 url.workspace = true
 uuid.workspace = true
 
-a2 = {version = "0.8", git = "https://github.com/mozilla-services/a2", branch="master"}
+a2 = { version = "0.8", git = "https://github.com/mozilla-services/a2", branch = "master" }
 bytebuffer = "2.1"
-again = { version = "0.1.2", default-features = false, features = ["log", "rand"] }
+again = { version = "0.1.2", default-features = false, features = [
+    "log",
+    "rand",
+] }
 async-trait = "0.1"
 autopush_common = { path = "../autopush-common" }
 jsonwebtoken = "8.0"
@@ -64,5 +67,8 @@ tempfile = "3.2.0"
 tokio = { workspace = true, features = ["fs", "macros"] }
 
 [features]
+default = ["dynamodb"]
+dynamodb = ["autopush_common/dynamodb"]
 emulator = ["bigtable"]
+dual = ["bigtable", "dynamodb"]
 bigtable = ["autopush_common/bigtable"]

autoendpoint/src/server.rs

@@ -9,12 +9,17 @@ use actix_web::{
 };
 #[cfg(feature = "bigtable")]
 use autopush_common::db::bigtable::BigTableClientImpl;
+#[cfg(feature = "dual")]
+use autopush_common::db::dual::DualClientImpl;
 use cadence::StatsdClient;
 use fernet::MultiFernet;
 use serde_json::json;
 
+#[cfg(feature = "dynamodb")]
+use autopush_common::db::dynamodb::DdbClientImpl;
+
 use autopush_common::{
-    db::{client::DbClient, dynamodb::DdbClientImpl, DbSettings, StorageType},
+    db::{client::DbClient, DbSettings, StorageType},
     middleware::sentry::SentryWrapper,
 };
 
@@ -63,6 +68,7 @@ impl Server {
             },
         };
         let db: Box<dyn DbClient> = match StorageType::from_dsn(&db_settings.dsn) {
+            #[cfg(feature = "dynamodb")]
             StorageType::DynamoDb => {
                 debug!("Using Dynamodb");
                 Box::new(DdbClientImpl::new(metrics.clone(), &db_settings)?)
@@ -72,6 +78,8 @@ impl Server {
                 debug!("Using BigTable");
                 Box::new(BigTableClientImpl::new(metrics.clone(), &db_settings)?)
             }
+            #[cfg(all(feature = "bigtable", feature = "dual"))]
+            StorageType::Dual => Box::new(DualClientImpl::new(metrics.clone(), &db_settings)?),
             _ => {
                 debug!("No idea what {:?} is", &db_settings.dsn);
                 return Err(ApiErrorKind::General(

autoendpoint/src/settings.rs

@@ -56,10 +56,10 @@ impl Default for Settings {
             db_settings: "".to_owned(),
             router_table_name: "router".to_string(),
             message_table_name: "message".to_string(),
-            /// max data is a bit hard to figure out, due to encryption. Using something
-            /// like pywebpush, if you encode a block of 4096 bytes, you'll get a
-            /// 4216 byte data block. Since we're going to be receiving this, we have to
-            /// presume base64 encoding, so we can bump things up to 5630 bytes max.
+            // max data is a bit hard to figure out, due to encryption. Using something
+            // like pywebpush, if you encode a block of 4096 bytes, you'll get a
+            // 4216 byte data block. Since we're going to be receiving this, we have to
+            // presume base64 encoding, so we can bump things up to 5630 bytes max.
             max_data_bytes: 5630,
             crypto_keys: format!("[{}]", Fernet::generate_key()),
             auth_keys: r#"["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB="]"#.to_string(),