From c00cb440dcbef0d45a60a3f3d2f2073193a9dff0 Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Thu, 3 Aug 2023 10:21:52 -0700
Subject: [PATCH 01/11] feat: support dual mode data storage

Allow for dual mode, where data is written to the primary, but the
secondary is read in order to 'back fill'. For simplicity, Dual presumes
that the primary data store is Bigtable and the secondary is DynamoDb.

This presumes that the `db_settings` contains serialized settings for
both data stores.

Closes: SYNC-3452
---
 .circleci/config.yml                          |  32 +-
 Cargo.lock                                    | 549 +++++++++---------
 Cargo.toml                                    |  14 +-
 autoconnect/Cargo.toml                        |   2 +-
 autoendpoint/Cargo.toml                       |   3 +
 autoendpoint/src/server.rs                    |  10 +-
 autopush-common/Cargo.toml                    |   7 +-
 .../src/db/bigtable/bigtable_client/mod.rs    |  17 +-
 autopush-common/src/db/mod.rs                 |  15 +-
 docs/src/fcm.md                               |   2 +-
 10 files changed, 344 insertions(+), 307 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 360dea814..c141f00d6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -62,6 +62,11 @@ jobs:
           username: $DOCKER_USER
           password: $DOCKER_PASS
         command: -jar DynamoDBLocal.jar -sharedDb
+      - image: spotify/bigtable-emulator
+        auth:
+          username: $DOCKER_USER
+          password: $DOCKER_PASS
+
     resource_class: xlarge
     environment:
       AWS_LOCAL_DYNAMODB: http://localhost:8000
@@ -97,6 +102,27 @@ jobs:
             echo 'export PATH=$PATH:$HOME/.cargo/bin' >> $BASH_ENV
             rustc --version
             cargo build --features=bigtable
+      - run:
+          name: Setup bigtable
+          environment:
+            BIGTABLE_EMULATOR_HOST: localhost:8080
+          command: |
+            apt update
+            apt install gnupg curl -y
+            # Fetch the key for google.
+            echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+            curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add -
+            # refresh to get the packages
+            apt-get update
+            apt-get install google-cloud-cli-cbt -y
+            # create the baseline (for unit tests)
+            cbt -project test -instance test createtable autopush
+            cbt -project test -instance test createfamily autopush message
+            cbt -project test -instance test createfamily autopush message_topic
+            cbt -project test -instance test createfamily autopush router
+            cbt -project test -instance test setgcpolicy autopush message maxage=1s
+            cbt -project test -instance test setgcpolicy autopush message_topic maxversions=1
+            cbt -project test -instance test setgcpolicy autopush router maxversions=1
       - run:
           name: Check formatting
           command: |
@@ -105,13 +131,15 @@ jobs:
       - run:
           name: Rust tests
           environment:
-            BIGTABLE_EMULATOR_HOST: localhost:8086
+            BIGTABLE_EMULATOR_HOST: localhost:8080
+            AUTOCONNECT__DB_DSN: dual
+            AUTOCONNECT__DB_SETTINGS: "{\"primary\":{\"db_settings\":\"{\\\"message_family\\\":\\\"message\\\",\\\"router_family\\\":\\\"router\\\",\\\"message_topic_family\\\":\\\"message_topic\\\",\\\"table_name\\\":\\\"projects/test/instances/test/tables/autopush\\\"}\",\"dsn\":\"grpc://locahost:8080\"},\"secondary\":{\"db_settings\":\"{\\\"message_table\\\":\\\"test_message\\\",\\\"router_table\\\":\\\"test_router\\\"}\",\"dsn\":\"http://localhost:8000/\"}}"
           # Note: This build can potentially exceed the amount of memory availble to the CircleCI instance.
           # We've seen that limiting the number of jobs helps reduce the frequency of this. (Note that
           # when doing discovery, we found that the docker image `meminfo` and `cpuinfo` often report
           # the machine level memory and CPU which are far higher than the memory allocated to the docker
           # instance. This may be causing rust to be overly greedy triggering the VM to OOM the process.)
-          command: cargo test --features=bigtable --jobs=8
+          command: cargo test --jobs=8 --features=dual --features=emulator
       - run:
           name: Integration tests (Autopush Legacy)
           command: |
diff --git a/Cargo.lock b/Cargo.lock
index e2a3de5a5..0876f10ef 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -98,7 +98,7 @@ dependencies = [
  "http 0.2.9",
  "httparse",
  "httpdate",
- "itoa 1.0.8",
+ "itoa 1.0.9",
  "language-tags",
  "local-channel",
  "mime",
@@ -140,12 +140,12 @@ dependencies = [
 
 [[package]]
 name = "actix-macros"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6"
+checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
 dependencies = [
- "quote 1.0.29",
- "syn 1.0.109",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -276,7 +276,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "http 0.2.9",
- "itoa 1.0.8",
+ "itoa 1.0.9",
  "language-tags",
  "log",
  "mime",
@@ -288,7 +288,7 @@ dependencies = [
  "serde_urlencoded 0.7.1",
  "smallvec 1.11.0",
  "socket2",
- "time 0.3.23",
+ "time 0.3.25",
  "url 2.4.0",
 ]
 
@@ -299,8 +299,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9"
 dependencies = [
  "actix-router",
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "syn 1.0.109",
 ]
 
@@ -323,8 +323,8 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d44b8fee1ced9671ba043476deddef739dd0959bf77030b26b738cc591737a7"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "syn 1.0.109",
 ]
 
@@ -427,9 +427,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.71"
+version = "1.0.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
+checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
 
 [[package]]
 name = "arc-swap"
@@ -476,20 +476,20 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "async-trait"
-version = "0.1.71"
+version = "0.1.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf"
+checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -520,7 +520,7 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "autoconnect"
-version = "1.67.3"
+version = "1.67.4"
 dependencies = [
  "actix",
  "actix-cors",
@@ -550,7 +550,7 @@ dependencies = [
  "reqwest 0.11.18",
  "sentry",
  "sentry-actix",
- "sentry-core 0.29.3",
+ "sentry-core",
  "serde",
  "serde_derive",
  "serde_json",
@@ -561,12 +561,12 @@ dependencies = [
  "slog-scope",
  "slog-stdlog",
  "slog-term",
- "uuid 1.4.0",
+ "uuid 1.4.1",
 ]
 
 [[package]]
 name = "autoconnect_common"
-version = "1.67.3"
+version = "1.67.4"
 dependencies = [
  "actix-web",
  "autopush_common",
@@ -580,12 +580,12 @@ dependencies = [
  "slog",
  "slog-scope",
  "tokio 1.29.1",
- "uuid 1.4.0",
+ "uuid 1.4.1",
 ]
 
 [[package]]
 name = "autoconnect_settings"
-version = "1.67.3"
+version = "1.67.4"
 dependencies = [
  "autoconnect_common",
  "autopush_common",
@@ -604,7 +604,7 @@ dependencies = [
 
 [[package]]
 name = "autoconnect_web"
-version = "1.67.3"
+version = "1.67.4"
 dependencies = [
  "actix-codec",
  "actix-http",
@@ -625,12 +625,12 @@ dependencies = [
  "serde_json",
  "slog-scope",
  "tokio 1.29.1",
- "uuid 1.4.0",
+ "uuid 1.4.1",
 ]
 
 [[package]]
 name = "autoconnect_ws"
-version = "1.67.3"
+version = "1.67.4"
 dependencies = [
  "actix-http",
  "actix-rt",
@@ -656,7 +656,7 @@ dependencies = [
 
 [[package]]
 name = "autoconnect_ws_sm"
-version = "1.67.3"
+version = "1.67.4"
 dependencies = [
  "actix-rt",
  "actix-web",
@@ -674,12 +674,12 @@ dependencies = [
  "slog-scope",
  "thiserror",
  "tokio 1.29.1",
- "uuid 1.4.0",
+ "uuid 1.4.1",
 ]
 
 [[package]]
 name = "autoendpoint"
-version = "1.67.3"
+version = "1.67.4"
 dependencies = [
  "a2",
  "actix-cors",
@@ -712,7 +712,7 @@ dependencies = [
  "rusoto_dynamodb 0.47.0",
  "sentry",
  "sentry-actix",
- "sentry-core 0.30.0",
+ "sentry-core",
  "serde",
  "serde_derive",
  "serde_dynamodb 0.9.0",
@@ -728,7 +728,7 @@ dependencies = [
  "thiserror",
  "tokio 1.29.1",
  "url 2.4.0",
- "uuid 1.4.0",
+ "uuid 1.4.1",
  "validator",
  "validator_derive",
  "yup-oauth2",
@@ -736,7 +736,7 @@ dependencies = [
 
 [[package]]
 name = "autopush"
-version = "1.67.3"
+version = "1.67.4"
 dependencies = [
  "autopush_common",
  "base64 0.21.2",
@@ -784,13 +784,13 @@ dependencies = [
  "tokio-openssl",
  "tokio-tungstenite",
  "tungstenite",
- "uuid 1.4.0",
+ "uuid 1.4.1",
  "woothee",
 ]
 
 [[package]]
 name = "autopush_common"
-version = "1.67.3"
+version = "1.67.4"
 dependencies = [
  "actix-http",
  "actix-rt",
@@ -826,7 +826,7 @@ dependencies = [
  "rusoto_dynamodb 0.47.0",
  "sentry",
  "sentry-backtrace",
- "sentry-core 0.30.0",
+ "sentry-core",
  "serde",
  "serde_derive",
  "serde_dynamodb 0.9.0",
@@ -845,7 +845,7 @@ dependencies = [
  "tokio-core",
  "tungstenite",
  "url 2.4.0",
- "uuid 1.4.0",
+ "uuid 1.4.1",
  "woothee",
 ]
 
@@ -871,7 +871,7 @@ dependencies = [
  "futures-util",
  "h2 0.3.20",
  "http 0.2.9",
- "itoa 1.0.8",
+ "itoa 1.0.9",
  "log",
  "mime",
  "percent-encoding 2.3.0",
@@ -946,8 +946,8 @@ dependencies = [
  "lazycell",
  "log",
  "peeking_take_while",
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "regex",
  "rustc-hash",
  "shlex 1.1.0",
@@ -1116,11 +1116,12 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.79"
+version = "1.0.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+checksum = "6c6b2562119bf28c3439f7f02db99faf0aa1a8cdfe5772a2ee155d32227239f0"
 dependencies = [
  "jobserver",
+ "libc",
 ]
 
 [[package]]
@@ -1263,7 +1264,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb"
 dependencies = [
  "percent-encoding 2.3.0",
- "time 0.3.23",
+ "time 0.3.25",
  "version_check",
 ]
 
@@ -1446,12 +1447,12 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "0.1.26"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096"
+checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b"
 dependencies = [
- "quote 1.0.29",
- "syn 1.0.109",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -1495,7 +1496,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
 dependencies = [
  "serde",
- "uuid 1.4.0",
+ "uuid 1.4.1",
+]
+
+[[package]]
+name = "deranged"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929"
+dependencies = [
+ "serde",
 ]
 
 [[package]]
@@ -1505,8 +1515,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
 dependencies = [
  "convert_case",
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "rustc_version 0.4.0",
  "syn 1.0.109",
 ]
@@ -1623,9 +1633,9 @@ checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
 
 [[package]]
 name = "either"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
 [[package]]
 name = "encoding_rs"
@@ -1664,18 +1674,18 @@ dependencies = [
 
 [[package]]
 name = "erased-serde"
-version = "0.3.27"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f94c0e13118e7d7533271f754a168ae8400e6a1cc043f2bfd53cc7290f1a1de3"
+checksum = "da96524cc884f6558f1769b6c46686af2fe8e8b4cd253bd5a3cdba8181b8e070"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "errno"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
 dependencies = [
  "errno-dragonfly",
  "libc",
@@ -1708,8 +1718,8 @@ version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "syn 1.0.109",
  "synstructure",
 ]
@@ -1722,12 +1732,9 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
 
 [[package]]
 name = "fastrand"
-version = "1.9.0"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"
-dependencies = [
- "instant",
-]
+checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
 
 [[package]]
 name = "fernet"
@@ -1940,9 +1947,9 @@ version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -2050,9 +2057,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "google-cloud-rust-raw"
-version = "0.14.0"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1887de8efd052e35bf75e4ed4bc78de35b69447a4b6d9f2e7ede52579512f318"
+checksum = "fbabcfb0bcc5e3222191c3f0fba962b0cbf4242d2effe2a865090eee492ffc9c"
 dependencies = [
  "futures 0.3.28",
  "grpcio",
@@ -2061,9 +2068,9 @@ dependencies = [
 
 [[package]]
 name = "grpcio"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55edb2eaa4b13b9a8a07f1a1bbaadc8a5bf4132c2b1fa6358d2be14afd18a2c3"
+checksum = "609832ca501baeb662dc81932fda9ed83f5d058f4b899a807ba222ce696f430a"
 dependencies = [
  "futures-executor",
  "futures-util",
@@ -2222,7 +2229,7 @@ checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
 dependencies = [
  "bytes 1.4.0",
  "fnv",
- "itoa 1.0.8",
+ "itoa 1.0.9",
 ]
 
 [[package]]
@@ -2311,7 +2318,7 @@ dependencies = [
  "http-body 0.4.5",
  "httparse",
  "httpdate",
- "itoa 1.0.8",
+ "itoa 1.0.9",
  "pin-project-lite 0.2.10",
  "socket2",
  "tokio 1.29.1",
@@ -2361,7 +2368,7 @@ dependencies = [
  "http 0.2.9",
  "hyper 0.14.27",
  "log",
- "rustls 0.21.5",
+ "rustls 0.21.6",
  "rustls-native-certs 0.6.3",
  "tokio 1.29.1",
  "tokio-rustls 0.24.1",
@@ -2488,17 +2495,6 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
-[[package]]
-name = "io-lifetimes"
-version = "1.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
-dependencies = [
- "hermit-abi 0.3.2",
- "libc",
- "windows-sys",
-]
-
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -2521,7 +2517,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
  "hermit-abi 0.3.2",
- "rustix 0.38.4",
+ "rustix",
  "windows-sys",
 ]
 
@@ -2542,9 +2538,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
 
 [[package]]
 name = "itoa"
-version = "1.0.8"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
 
 [[package]]
 name = "jobserver"
@@ -2635,9 +2631,9 @@ dependencies = [
 
 [[package]]
 name = "libz-sys"
-version = "1.1.9"
+version = "1.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db"
+checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b"
 dependencies = [
  "cc",
  "libc",
@@ -2653,15 +2649,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
-
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.3"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
+checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
 
 [[package]]
 name = "local-channel"
@@ -2884,8 +2874,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb"
 dependencies = [
  "cfg-if 1.0.0",
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "syn 1.0.109",
 ]
 
@@ -2986,9 +2976,9 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.15"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
 dependencies = [
  "autocfg 1.1.0",
 ]
@@ -3060,9 +3050,9 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -3186,9 +3176,9 @@ dependencies = [
 
 [[package]]
 name = "paste"
-version = "1.0.13"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4b27ab7be369122c218afc2079489cdcb4b517c0a3fc386ff11e1fedfcc2b35"
+checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
 
 [[package]]
 name = "pathdiff"
@@ -3225,9 +3215,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
 [[package]]
 name = "pest"
-version = "2.7.0"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f73935e4d55e2abf7f130186537b19e7a4abc886a0252380b59248af473a3fc9"
+checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a"
 dependencies = [
  "thiserror",
  "ucd-trie",
@@ -3235,9 +3225,9 @@ dependencies = [
 
 [[package]]
 name = "pest_derive"
-version = "2.7.0"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aef623c9bbfa0eedf5a0efba11a5ee83209c326653ca31ff019bec3a95bfff2b"
+checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853"
 dependencies = [
  "pest",
  "pest_generator",
@@ -3245,22 +3235,22 @@ dependencies = [
 
 [[package]]
 name = "pest_generator"
-version = "2.7.0"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3e8cba4ec22bada7fc55ffe51e2deb6a0e0db2d0b7ab0b103acc80d2510c190"
+checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929"
 dependencies = [
  "pest",
  "pest_meta",
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "pest_meta"
-version = "2.7.0"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a01f71cb40bd8bb94232df14b946909e14660e33fc05db3e50ae2a82d7ea0ca0"
+checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48"
 dependencies = [
  "once_cell",
  "pest",
@@ -3344,8 +3334,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
 dependencies = [
  "proc-macro-error-attr",
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "syn 1.0.109",
  "version_check",
 ]
@@ -3356,8 +3346,8 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "version_check",
 ]
 
@@ -3372,9 +3362,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.64"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da"
+checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
 dependencies = [
  "unicode-ident",
 ]
@@ -3406,11 +3396,11 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.29"
+version = "1.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
+checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965"
 dependencies = [
- "proc-macro2 1.0.64",
+ "proc-macro2 1.0.66",
 ]
 
 [[package]]
@@ -3672,9 +3662,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310"
+checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -4006,33 +3996,19 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
 dependencies = [
- "semver 1.0.17",
+ "semver 1.0.18",
 ]
 
 [[package]]
 name = "rustix"
-version = "0.37.23"
+version = "0.38.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06"
-dependencies = [
- "bitflags 1.3.2",
- "errno",
- "io-lifetimes",
- "libc",
- "linux-raw-sys 0.3.8",
- "windows-sys",
-]
-
-[[package]]
-name = "rustix"
-version = "0.38.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5"
+checksum = "1ee020b1716f0a80e2ace9b03441a749e402e86712f15f16fe8a8f75afac732f"
 dependencies = [
  "bitflags 2.3.3",
  "errno",
  "libc",
- "linux-raw-sys 0.4.3",
+ "linux-raw-sys",
  "windows-sys",
 ]
 
@@ -4063,9 +4039,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.21.5"
+version = "0.21.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36"
+checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb"
 dependencies = [
  "log",
  "ring",
@@ -4108,9 +4084,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e"
+checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59"
 dependencies = [
  "ring",
  "untrusted",
@@ -4118,15 +4094,15 @@ dependencies = [
 
 [[package]]
 name = "rustversion"
-version = "1.0.13"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc31bd9b61a32c31f9650d18add92aa83a49ba979c143eefd27fe7177b05bd5f"
+checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
 
 [[package]]
 name = "ryu"
-version = "1.0.14"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
 
 [[package]]
 name = "same-file"
@@ -4154,9 +4130,9 @@ checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28"
 
 [[package]]
 name = "scopeguard"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "sct"
@@ -4186,9 +4162,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
 
 [[package]]
 name = "security-framework"
-version = "2.9.1"
+version = "2.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8"
+checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
 dependencies = [
  "bitflags 1.3.2",
  "core-foundation",
@@ -4199,9 +4175,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.9.0"
+version = "2.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7"
+checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -4218,9 +4194,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"
+checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
 
 [[package]]
 name = "semver-parser"
@@ -4230,9 +4206,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 
 [[package]]
 name = "sentry"
-version = "0.30.0"
+version = "0.31.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5ce6d3512e2617c209ec1e86b0ca2fea06454cd34653c91092bf0f3ec41f8e3"
+checksum = "01b0ad16faa5d12372f914ed40d00bda21a6d1bdcc99264c5e5e1c9495cf3654"
 dependencies = [
  "httpdate",
  "log",
@@ -4240,120 +4216,103 @@ dependencies = [
  "reqwest 0.11.18",
  "sentry-backtrace",
  "sentry-contexts",
- "sentry-core 0.30.0",
+ "sentry-core",
  "sentry-debug-images",
  "sentry-panic",
+ "sentry-tracing",
  "tokio 1.29.1",
  "ureq",
 ]
 
 [[package]]
 name = "sentry-actix"
-version = "0.30.0"
+version = "0.31.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c210068218470670c03fbe59f53944061f9b1fcdb7748f9326248ab1ddf56238"
+checksum = "e9f81ee4dfb8e109fd99f0eb036fec548e66fd1db17a0224304c4a31ab0749ef"
 dependencies = [
  "actix-web",
  "futures-util",
- "sentry-core 0.30.0",
+ "sentry-core",
 ]
 
 [[package]]
 name = "sentry-backtrace"
-version = "0.30.0"
+version = "0.31.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e7fe408d4d1f8de188a9309916e02e129cbe51ca19e55badea5a64899399b1a"
+checksum = "11f2ee8f147bb5f22ac59b5c35754a759b9a6f6722402e2a14750b2a63fc59bd"
 dependencies = [
  "backtrace",
  "once_cell",
  "regex",
- "sentry-core 0.30.0",
+ "sentry-core",
 ]
 
 [[package]]
 name = "sentry-contexts"
-version = "0.30.0"
+version = "0.31.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5695096a059a89973ec541062d331ff4c9aeef9c2951416c894f0fff76340e7d"
+checksum = "dcd133362c745151eeba0ac61e3ba8350f034e9fe7509877d08059fe1d7720c6"
 dependencies = [
  "hostname",
  "libc",
  "os_info",
  "rustc_version 0.4.0",
- "sentry-core 0.30.0",
+ "sentry-core",
  "uname",
 ]
 
 [[package]]
 name = "sentry-core"
-version = "0.29.3"
+version = "0.31.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5acbd3da4255938cf0384b6b140e6c07ff65919c26e4d7a989d8d90ee88fa91"
-dependencies = [
- "once_cell",
- "rand 0.8.5",
- "sentry-types 0.29.3",
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "sentry-core"
-version = "0.30.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b22828bfd118a7b660cf7a155002a494755c0424cebb7061e4743ecde9c7dbc"
+checksum = "7163491708804a74446642ff2c80b3acd668d4b9e9f497f85621f3d250fd012b"
 dependencies = [
  "log",
  "once_cell",
  "rand 0.8.5",
- "sentry-types 0.30.0",
+ "sentry-types",
  "serde",
  "serde_json",
 ]
 
 [[package]]
 name = "sentry-debug-images"
-version = "0.30.0"
+version = "0.31.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a9164d44a2929b1b7670afd7e87552514b70d3ae672ca52884639373d912a3d"
+checksum = "6a5003d7ff08aa3b2b76994080b183e8cfa06c083e280737c9cee02ca1c70f5e"
 dependencies = [
  "findshlibs",
  "once_cell",
- "sentry-core 0.30.0",
+ "sentry-core",
 ]
 
 [[package]]
 name = "sentry-panic"
-version = "0.30.0"
+version = "0.31.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f4ced2a7a8c14899d58eec402d946f69d5ed26a3fc363a7e8b1e5cb88473a01"
+checksum = "c4dfe8371c9b2e126a8b64f6fefa54cef716ff2a50e63b5558a48b899265bccd"
 dependencies = [
  "sentry-backtrace",
- "sentry-core 0.30.0",
+ "sentry-core",
 ]
 
 [[package]]
-name = "sentry-types"
-version = "0.29.3"
+name = "sentry-tracing"
+version = "0.31.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10d8587b12c0b8211bb3066979ee57af6e8657e23cf439dc6c8581fd86de24e8"
+checksum = "5aca8b88978677a27ee1a91beafe4052306c474c06f582321fde72d2e2cc2f7f"
 dependencies = [
- "debugid",
- "getrandom 0.2.10",
- "hex",
- "serde",
- "serde_json",
- "thiserror",
- "time 0.3.23",
- "url 2.4.0",
- "uuid 1.4.0",
+ "sentry-backtrace",
+ "sentry-core",
+ "tracing-core",
+ "tracing-subscriber",
 ]
 
 [[package]]
 name = "sentry-types"
-version = "0.30.0"
+version = "0.31.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "360ee3270f7a4a1eee6c667f7d38360b995431598a73b740dfe420da548d9cc9"
+checksum = "9e7a88e0c1922d19b3efee12a8215f6a8a806e442e665ada71cc222cab72985f"
 dependencies = [
  "debugid",
  "getrandom 0.2.10",
@@ -4361,29 +4320,29 @@ dependencies = [
  "serde",
  "serde_json",
  "thiserror",
- "time 0.3.23",
+ "time 0.3.25",
  "url 2.4.0",
- "uuid 1.4.0",
+ "uuid 1.4.1",
 ]
 
 [[package]]
 name = "serde"
-version = "1.0.171"
+version = "1.0.180"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9"
+checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.171"
+version = "1.0.180"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682"
+checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -4409,11 +4368,11 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.102"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed"
+checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
 dependencies = [
- "itoa 1.0.8",
+ "itoa 1.0.9",
  "ryu",
  "serde",
 ]
@@ -4437,7 +4396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
 dependencies = [
  "form_urlencoded",
- "itoa 1.0.8",
+ "itoa 1.0.9",
  "ryu",
  "serde",
 ]
@@ -4515,9 +4474,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
 
 [[package]]
 name = "signal-hook"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9"
+checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
 dependencies = [
  "libc",
  "signal-hook-registry",
@@ -4547,7 +4506,7 @@ dependencies = [
  "num-bigint",
  "num-traits",
  "thiserror",
- "time 0.3.23",
+ "time 0.3.25",
 ]
 
 [[package]]
@@ -4639,7 +4598,7 @@ dependencies = [
  "slog",
  "term",
  "thread_local",
- "time 0.3.23",
+ "time 0.3.25",
 ]
 
 [[package]]
@@ -4721,8 +4680,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
 dependencies = [
  "heck 0.4.1",
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "rustversion",
  "syn 1.0.109",
 ]
@@ -4756,19 +4715,19 @@ version = "1.0.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "unicode-ident",
 ]
 
 [[package]]
 name = "syn"
-version = "2.0.25"
+version = "2.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2"
+checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "unicode-ident",
 ]
 
@@ -4778,8 +4737,8 @@ version = "0.12.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "syn 1.0.109",
  "unicode-xid 0.2.4",
 ]
@@ -4792,15 +4751,14 @@ checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60"
 
 [[package]]
 name = "tempfile"
-version = "3.6.0"
+version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6"
+checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998"
 dependencies = [
- "autocfg 1.1.0",
  "cfg-if 1.0.0",
  "fastrand",
  "redox_syscall 0.3.5",
- "rustix 0.37.23",
+ "rustix",
  "windows-sys",
 ]
 
@@ -4841,22 +4799,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.43"
+version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42"
+checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.43"
+version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f"
+checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -4882,11 +4840,12 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.23"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446"
+checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea"
 dependencies = [
- "itoa 1.0.8",
+ "deranged",
+ "itoa 1.0.9",
  "libc",
  "num_threads",
  "serde",
@@ -4902,9 +4861,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "time-macros"
-version = "0.2.10"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4"
+checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd"
 dependencies = [
  "time-core",
 ]
@@ -5068,8 +5027,8 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "syn 1.0.109",
 ]
 
@@ -5079,9 +5038,9 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -5171,7 +5130,7 @@ version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
 dependencies = [
- "rustls 0.21.5",
+ "rustls 0.21.6",
  "tokio 1.29.1",
 ]
 
@@ -5337,6 +5296,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
 dependencies = [
  "once_cell",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
+dependencies = [
+ "tracing-core",
 ]
 
 [[package]]
@@ -5411,9 +5380,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.10"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73"
+checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
 
 [[package]]
 name = "unicode-normalization"
@@ -5507,9 +5476,9 @@ dependencies = [
 
 [[package]]
 name = "uuid"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be"
+checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
 dependencies = [
  "getrandom 0.2.10",
  "serde",
@@ -5539,8 +5508,8 @@ dependencies = [
  "if_chain",
  "lazy_static",
  "proc-macro-error",
- "proc-macro2 1.0.64",
- "quote 1.0.29",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
  "regex",
  "syn 1.0.109",
  "validator_types",
@@ -5552,10 +5521,16 @@ version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "111abfe30072511849c5910134e8baf8dc05de4c0e5903d681cbd5c9c4d611e3"
 dependencies = [
- "proc-macro2 1.0.64",
+ "proc-macro2 1.0.66",
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "valuable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -5641,9 +5616,9 @@ dependencies = [
  "bumpalo",
  "log",
  "once_cell",
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
  "wasm-bindgen-shared",
 ]
 
@@ -5665,7 +5640,7 @@ version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
 dependencies = [
- "quote 1.0.29",
+ "quote 1.0.32",
  "wasm-bindgen-macro-support",
 ]
 
@@ -5675,9 +5650,9 @@ version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -5911,9 +5886,9 @@ dependencies = [
 
 [[package]]
 name = "xml-rs"
-version = "0.8.15"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a56c84a8ccd4258aed21c92f70c0f6dea75356b6892ae27c24139da456f9336"
+checksum = "47430998a7b5d499ccee752b41567bc3afc57e1327dc855b1a2aa44ce29b5fa1"
 
 [[package]]
 name = "yaml-rust"
@@ -5940,12 +5915,12 @@ dependencies = [
  "itertools",
  "log",
  "percent-encoding 2.3.0",
- "rustls 0.21.5",
+ "rustls 0.21.6",
  "rustls-pemfile",
  "seahash",
  "serde",
  "serde_json",
- "time 0.3.23",
+ "time 0.3.25",
  "tokio 1.29.1",
  "tower-service",
  "url 2.4.0",
@@ -5966,25 +5941,25 @@ version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
- "proc-macro2 1.0.64",
- "quote 1.0.29",
- "syn 2.0.25",
+ "proc-macro2 1.0.66",
+ "quote 1.0.32",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "zstd"
-version = "0.12.3+zstd.1.5.2"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806"
+checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"
 dependencies = [
  "zstd-safe",
 ]
 
 [[package]]
 name = "zstd-safe"
-version = "6.0.5+zstd.1.5.4"
+version = "6.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b"
+checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"
 dependencies = [
  "libc",
  "zstd-sys",
diff --git a/Cargo.toml b/Cargo.toml
index e8a24576c..77fefee3a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -12,7 +12,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.67.3"
+version = "1.67.4"
 authors = [
   "Ben Bangert <ben@groovie.org>",
   "JR Conlin <jrconlin@mozilla.com>",
@@ -40,7 +40,7 @@ bytestring = "1.2"
 cadence = "0.29"
 chrono = "0.4"
 config = "0.13"
-ctor = "0.1"
+ctor = "0.2"
 docopt = "1.1"
 env_logger = "0.10"
 fernet = "0.2.0"
@@ -61,10 +61,10 @@ reqwest = {version="0.11", features = ["json"] }
 rusoto_core = { version="0.47", default-features=false, features=["rustls"] } # locked by serde_dynamodb 0.9
 rusoto_credential = { version="0.47"} # locked by serde_dynamodb 0.9
 rusoto_dynamodb = { version="0.47", default-features=false, features=["rustls"]} # locked by serde_dynamodb 0.9
-sentry = { version = "0.30", features = ["debug-logs"] } # Using debug-logs avoids https://github.com/getsentry/sentry-rust/issues/237
-sentry-core = {version = "0.30"}
-sentry-actix = "0.30"
-sentry-backtrace = "0.30"
+sentry = { version = "0.31", features = ["debug-logs"] } # Using debug-logs avoids https://github.com/getsentry/sentry-rust/issues/237
+sentry-core = {version = "0.31", default-features = false, features=["client"] }
+sentry-actix = "0.31"
+sentry-backtrace = "0.31"
 serde = "1.0"
 serde_derive = "1.0"
 serde_dynamodb = "0.9"
@@ -93,7 +93,7 @@ autoconnect_settings = { path = "./autoconnect/autoconnect-settings" }
 autoconnect_web = { path = "./autoconnect/autoconnect-web" }
 autoconnect_ws = { path = "./autoconnect/autoconnect-ws" }
 autoconnect_ws_clientsm = { path ="./autoconnect/autoconnect-ws/autoconnect-ws-clientsm" }
-autopush_common = { path = "./autopush-common", features=["bigtable"]}
+autopush_common = { path = "./autopush-common", features=["dynamodb", "bigtable"]}
 
 [profile.release]
 debug = 1
diff --git a/autoconnect/Cargo.toml b/autoconnect/Cargo.toml
index 563f7883d..7250161e4 100644
--- a/autoconnect/Cargo.toml
+++ b/autoconnect/Cargo.toml
@@ -51,7 +51,7 @@ docopt = "1.1"
 # for sentry middleware
 
 futures-util = { version = "0.3.5", default-features = false }
-sentry-core = { version = "0.29.1", default-features = false, features = ["client"] }
+sentry-core = { workspace=true, default-features = false, features = ["client"] }
 
 [features]
 emulator = ["bigtable"]
diff --git a/autoendpoint/Cargo.toml b/autoendpoint/Cargo.toml
index 09d8bd2b6..f7bdc93da 100644
--- a/autoendpoint/Cargo.toml
+++ b/autoendpoint/Cargo.toml
@@ -65,5 +65,8 @@ tempfile = "3.2.0"
 tokio = { workspace = true, features = ["fs", "macros"] }
 
 [features]
+default = ["dynamodb"]
+dynamodb = []
 emulator = ["bigtable"]
+dual = ["bigtable"]
 bigtable = ["autopush_common/bigtable"]
diff --git a/autoendpoint/src/server.rs b/autoendpoint/src/server.rs
index 0ad951244..1b6c506ee 100644
--- a/autoendpoint/src/server.rs
+++ b/autoendpoint/src/server.rs
@@ -9,12 +9,17 @@ use actix_web::{
 };
 #[cfg(feature = "bigtable")]
 use autopush_common::db::bigtable::BigTableClientImpl;
+#[cfg(feature = "dual")]
+use autopush_common::db::dual::DualClientImpl;
 use cadence::StatsdClient;
 use fernet::MultiFernet;
 use serde_json::json;
 
+#[cfg(feature = "dynamodb")]
+use autopush_common::db::dynamodb::DdbClientImpl;
+
 use autopush_common::{
-    db::{client::DbClient, dynamodb::DdbClientImpl, DbSettings, StorageType},
+    db::{client::DbClient, DbSettings, StorageType},
     middleware::sentry::SentryWrapper,
 };
 
@@ -63,6 +68,7 @@ impl Server {
             },
         };
         let db: Box<dyn DbClient> = match StorageType::from_dsn(&db_settings.dsn) {
+            #[cfg(feature = "dynamodb")]
             StorageType::DynamoDb => {
                 debug!("Using Dynamodb");
                 Box::new(DdbClientImpl::new(metrics.clone(), &db_settings)?)
@@ -72,6 +78,8 @@ impl Server {
                 debug!("Using BigTable");
                 Box::new(BigTableClientImpl::new(metrics.clone(), &db_settings)?)
             }
+            #[cfg(all(feature = "bigtable", feature = "dual"))]
+            StorageType::Dual => Box::new(DualClientImpl::new(metrics.clone(), &db_settings)?),
             _ => {
                 debug!("No idea what {:?} is", &db_settings.dsn);
                 return Err(ApiErrorKind::General(
diff --git a/autopush-common/Cargo.toml b/autopush-common/Cargo.toml
index fed0ecb4e..8d7610d16 100644
--- a/autopush-common/Cargo.toml
+++ b/autopush-common/Cargo.toml
@@ -61,8 +61,9 @@ woothee = "0.13"
 
 # #[cfg(bigtable)] for this section.
 # the following three crates must match what is specifed in google-cloud-rust-raw's dependencies
-google-cloud-rust-raw = {version="0.14"} # features=["bigtable"]}
-grpcio = {version="=0.12.0", features=["openssl"]}
+google-cloud-rust-raw = {version="0.15", default-features=false, features=["bigtable"]}
+#google-cloud-rust-raw = {path = "../../google-cloud-rust/googleapis-raw", version="0.15", default-features=false, features=["bigtable"]}
+grpcio = {version="=0.12.1", features=["openssl"]}
 protobuf = "=2.28.0"  # grpcio does not support protobuf 3+
 
 [dev-dependencies]
@@ -74,6 +75,8 @@ actix-rt = "2.8"
 [features]
 # for testing:
 # default = ["emulator"]
+default = ["dynamodb"]
+dynamodb = []
 bigtable = []
 emulator = ["bigtable"]  # used for testing big table, requires an external bigtable emulator running.
 
diff --git a/autopush-common/src/db/bigtable/bigtable_client/mod.rs b/autopush-common/src/db/bigtable/bigtable_client/mod.rs
index 8e090cc73..b81564b55 100644
--- a/autopush-common/src/db/bigtable/bigtable_client/mod.rs
+++ b/autopush-common/src/db/bigtable/bigtable_client/mod.rs
@@ -951,9 +951,9 @@ impl DbClient for BigTableClientImpl {
             );
             regex_filter
         });
-        // Note set_rows_limit(v) limits the returned results from Bigtable.
-        // If you're doing additional filtering later, this may not be what
-        // you want and may artificially truncate possible return sets.
+        // Note set_rows_limit(v) limits the returned results
+        // If you're doing additional filtering later, this is not what
+        // you want.
         /*
         if limit > 0 {
             trace!("🉑 Setting limit to {limit}");
@@ -1090,10 +1090,17 @@ mod tests {
     }
 
     fn new_client() -> DbResult<BigTableClientImpl> {
+        let env_dsn = format!(
+            "grpc://{}",
+            std::env::var("BIGTABLE_EMULATOR_HOST").unwrap_or("localhost:8080".to_owned())
+        );
         let settings = DbSettings {
             // this presumes the table was created with
-            // `cbt -project test -instance test createtable autopush`
-            dsn: Some("grpc://localhost:8086".to_owned()),
+            // ```
+            // cbt -project test -instance test createtable autopush
+            // ```
+            // with `message`, `router`, and `message_topic` families
+            dsn: Some(env_dsn),
             db_settings: json!({"table_name":"projects/test/instances/test/tables/autopush"})
                 .to_string(),
         };
diff --git a/autopush-common/src/db/mod.rs b/autopush-common/src/db/mod.rs
index e5efddc19..e3b10427e 100644
--- a/autopush-common/src/db/mod.rs
+++ b/autopush-common/src/db/mod.rs
@@ -23,6 +23,9 @@ use crate::db::{dynamodb::has_connected_this_month, util::generate_last_connect}
 #[cfg(feature = "bigtable")]
 pub mod bigtable;
 pub mod client;
+#[cfg(all(feature = "bigtable", feature = "dynamodb"))]
+pub mod dual;
+#[cfg(feature = "dynamodb")]
 pub mod dynamodb;
 pub mod error;
 pub mod models;
@@ -46,7 +49,11 @@ pub enum StorageType {
     INVALID,
     #[cfg(feature = "bigtable")]
     BigTable,
+    #[cfg(feature = "dynamodb")]
     DynamoDb,
+    #[cfg(all(feature = "bigtable", feature = "dynamodb"))]
+    Dual,
+    // Postgres,
 }
 
 /// The type of storage to use.
@@ -69,6 +76,7 @@ impl StorageType {
         let dsn = dsn
             .clone()
             .unwrap_or(std::env::var("AWS_LOCAL_DYNAMODB").unwrap_or_default());
+        #[cfg(feature = "dynamodb")]
         if dsn.starts_with("http") {
             trace!("Found http");
             return Self::DynamoDb;
@@ -86,13 +94,18 @@ impl StorageType {
             }
             return Self::BigTable;
         }
+        #[cfg(all(feature = "bigtable", feature = "dynamodb"))]
+        if dsn.to_lowercase() == "dual" {
+            trace!("Found Dual mode");
+            return Self::Dual;
+        }
         Self::INVALID
     }
 }
 
 /// The universal settings for the database
 /// abstractor.
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, Deserialize)]
 pub struct DbSettings {
     /// Database connector string
     pub dsn: Option<String>,
diff --git a/docs/src/fcm.md b/docs/src/fcm.md
index 976c4e496..2d3518c0e 100644
--- a/docs/src/fcm.md
+++ b/docs/src/fcm.md
@@ -1,3 +1,3 @@
 # Google GCM/FCM
 
-<!-- TODO: finish this doc --> 
\ No newline at end of file
+<!-- TODO: finish this doc -->
\ No newline at end of file

From 4ed4fbdfc7cd2ddffe4bba46d5f01aeab2d64855 Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Thu, 3 Aug 2023 16:34:49 -0700
Subject: [PATCH 02/11] f pick up spare

---
 autopush-common/src/db/dual/mod.rs | 254 +++++++++++++++++++++++++++++
 1 file changed, 254 insertions(+)
 create mode 100644 autopush-common/src/db/dual/mod.rs

diff --git a/autopush-common/src/db/dual/mod.rs b/autopush-common/src/db/dual/mod.rs
new file mode 100644
index 000000000..48afd85e2
--- /dev/null
+++ b/autopush-common/src/db/dual/mod.rs
@@ -0,0 +1,254 @@
+//! Dual data store.
+//!
+//! This uses two data stores, a primary and a secondary, and if a
+//! read operation fails for the primary, the secondary is automatically
+//! used. All write operations ONLY go to the primary.
+//!
+//! This requires both the `dynamodb` and `bigtable` features.
+//!
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use cadence::StatsdClient;
+use serde::Deserialize;
+use serde_json::from_str;
+use uuid::Uuid;
+
+use crate::db::{
+    client::{DbClient, FetchMessageResponse},
+    error::{DbError, DbResult},
+    DbSettings, Notification, User,
+};
+
+use super::StorageType;
+
+use crate::db::bigtable::BigTableClientImpl;
+use crate::db::dynamodb::DdbClientImpl;
+
+#[derive(Clone)]
+pub struct DualClientImpl {
+    primary: BigTableClientImpl,
+    secondary: DdbClientImpl,
+    _metrics: Arc<StatsdClient>,
+}
+
+#[derive(Clone, Debug, Default, Deserialize)]
+pub struct DualDbSettings {
+    #[serde(default)]
+    primary: DbSettings,
+    #[serde(default)]
+    secondary: DbSettings,
+}
+
+impl DualClientImpl {
+    pub fn new(metrics: Arc<StatsdClient>, settings: &DbSettings) -> DbResult<Self> {
+        // Not really sure we need the dsn here.
+        let db_settings: DualDbSettings = from_str(&settings.db_settings).map_err(|e| {
+            DbError::General(format!("Could not parse DualDBSettings string {:?}", e))
+        })?;
+        if StorageType::from_dsn(&db_settings.primary.dsn) != StorageType::BigTable {
+            return Err(DbError::General(
+                "Invalid primary DSN specified (must be BigTable type)".to_owned(),
+            ));
+        }
+        if StorageType::from_dsn(&db_settings.secondary.dsn) != StorageType::DynamoDb {
+            return Err(DbError::General(
+                "Invalid secondary DSN specified (must be DynamoDB type)".to_owned(),
+            ));
+        }
+        let primary = BigTableClientImpl::new(metrics.clone(), &db_settings.primary)?;
+        let secondary = DdbClientImpl::new(metrics.clone(), &db_settings.secondary)?;
+        Ok(Self {
+            primary,
+            secondary,
+            _metrics: metrics,
+        })
+    }
+}
+
+#[async_trait]
+impl DbClient for DualClientImpl {
+    async fn add_user(&self, user: &User) -> DbResult<()> {
+        self.primary.add_user(user).await
+    }
+
+    async fn update_user(&self, user: &User) -> DbResult<()> {
+        self.primary.update_user(user).await
+    }
+
+    async fn get_user(&self, uaid: &Uuid) -> DbResult<Option<User>> {
+        match self.primary.get_user(uaid).await {
+            Ok(Some(user)) => Ok(Some(user)),
+            Ok(None) => {
+                if let Ok(Some(user)) = self.secondary.get_user(uaid).await {
+                    // copy the user record over to the new data store.
+                    self.primary.add_user(&user).await?;
+                    return Ok(Some(user));
+                }
+                Ok(None)
+            }
+            Err(err) => Err(err),
+        }
+    }
+
+    async fn remove_user(&self, uaid: &Uuid) -> DbResult<()> {
+        let result = self.primary.remove_user(uaid).await?;
+        // try removing the user from the old store, just in case.
+        // leaving them could cause false reporting later.
+        let _ = self.secondary.remove_user(uaid).await;
+        Ok(result)
+    }
+
+    async fn add_channel(&self, uaid: &Uuid, channel_id: &Uuid) -> DbResult<()> {
+        self.primary.add_channel(uaid, channel_id).await
+    }
+
+    async fn get_channels(&self, uaid: &Uuid) -> DbResult<HashSet<Uuid>> {
+        let mut channels = self.primary.get_channels(uaid).await?;
+        // check to see if we need to copy over channels from the secondary
+        if channels.is_empty() {
+            channels = self.secondary.get_channels(uaid).await?;
+        }
+        Ok(channels)
+    }
+
+    async fn remove_channel(&self, uaid: &Uuid, channel_id: &Uuid) -> DbResult<bool> {
+        let presult = self.primary.remove_channel(uaid, channel_id).await?;
+        let sresult = self.secondary.remove_channel(uaid, channel_id).await?;
+        Ok(presult | sresult)
+    }
+
+    async fn remove_node_id(&self, uaid: &Uuid, node_id: &str, connected_at: u64) -> DbResult<()> {
+        let _ = self
+            .secondary
+            .remove_node_id(uaid, node_id, connected_at)
+            .await;
+        self.primary
+            .remove_node_id(uaid, node_id, connected_at)
+            .await
+    }
+
+    async fn save_message(&self, uaid: &Uuid, message: Notification) -> DbResult<()> {
+        self.primary.save_message(uaid, message).await
+    }
+
+    async fn remove_message(&self, uaid: &Uuid, sort_key: &str) -> DbResult<()> {
+        let _ = self.secondary.remove_message(uaid, sort_key).await;
+        self.primary.remove_message(uaid, sort_key).await
+    }
+
+    async fn fetch_topic_messages(
+        &self,
+        uaid: &Uuid,
+        limit: usize,
+    ) -> DbResult<FetchMessageResponse> {
+        let result = self.primary.fetch_topic_messages(uaid, limit).await?;
+        if result.messages.is_empty() {
+            return self.secondary.fetch_topic_messages(uaid, limit).await;
+        }
+        return Ok(result);
+    }
+
+    async fn fetch_timestamp_messages(
+        &self,
+        uaid: &Uuid,
+        timestamp: Option<u64>,
+        limit: usize,
+    ) -> DbResult<FetchMessageResponse> {
+        let result = self
+            .primary
+            .fetch_timestamp_messages(uaid, timestamp, limit)
+            .await?;
+        if result.messages.is_empty() {
+            return self
+                .secondary
+                .fetch_timestamp_messages(uaid, timestamp, limit)
+                .await;
+        }
+        return Ok(result);
+    }
+
+    async fn save_messages(&self, uaid: &Uuid, messages: Vec<Notification>) -> DbResult<()> {
+        self.primary.save_messages(uaid, messages).await
+    }
+
+    async fn increment_storage(&self, uaid: &Uuid, timestamp: u64) -> DbResult<()> {
+        self.primary.increment_storage(uaid, timestamp).await
+    }
+
+    async fn health_check(&self) -> DbResult<bool> {
+        Ok(self.primary.health_check().await? && self.secondary.health_check().await?)
+    }
+
+    async fn router_table_exists(&self) -> DbResult<bool> {
+        self.primary.router_table_exists().await
+    }
+
+    async fn message_table_exists(&self) -> DbResult<bool> {
+        self.primary.message_table_exists().await
+    }
+
+    fn rotating_message_table(&self) -> Option<&str> {
+        None
+    }
+
+    fn box_clone(&self) -> Box<dyn DbClient> {
+        Box::new(self.clone())
+    }
+}
+
+#[cfg(all(test, feature = "bigtable", feature = "dynamodb"))]
+mod test {
+    use super::*;
+    use cadence::{NopMetricSink, StatsdClient};
+    use serde_json::json;
+
+    /// This test checks the dual parser, but also serves as a bit of
+    /// documentation for how the db_settings argument should be structured
+    #[test]
+    fn test_args() -> DbResult<()> {
+        let arg_str = json!({
+            "primary": {
+                "dsn": "grpc://bigtable.googleapis.com",  // Note that this is the general endpoint.
+                "db_settings": json!({
+                    "table_name": "projects/some-project/instances/some-instance/tables/some-table", // Note the full path.
+                    "message_family": "messageFamily",
+                    "router_family": "routerFamily",
+                }).to_string(),
+            },
+            "secondary": {
+                "dsn": "http://localhost:8000/",
+                "db_settings": json!({
+                    "router_table": "test_router",
+                    "message_table": "test_message",
+                }).to_string(),
+            },
+        })
+        .to_string();
+
+        // the output string looks like:
+        /*
+        "{\"primary\":{\"db_settings\":\"{\\\"message_family\\\":\\\"message\\\",\\\"router_family\\\":\\\"router\\\",\\\"table_name\\\":\\\"projects/some-project/instances/some-instance/tables/some-table\\\"}\",\"dsn\":\"grpc://bigtable.googleapis.com\"},\"secondary\":{\"db_settings\":\"{\\\"message_table\\\":\\\"test_message\\\",\\\"router_table\\\":\\\"test_router\\\"}\",\"dsn\":\"http://localhost:8000/\"}}"
+         */
+        // some additional escaping may be required to encode this as an environment variable.
+        // not all variables are required, feel free to liberally use fall-back defaults.
+
+        // dbg!(&arg_str);
+
+        let dual_settings = DbSettings {
+            dsn: Some("dual".to_owned()),
+            db_settings: arg_str,
+        };
+        let metrics = Arc::new(StatsdClient::builder("", NopMetricSink).build());
+
+        // this is the actual test. It should create `dual` without raising an error
+        // Specify "BIGTABLE_EMULATOR_HOST" to skip credential check for emulator.
+        let dual = DualClientImpl::new(metrics, &dual_settings)?;
+
+        // for sanity sake, make sure the passed message_family isn't the default value.
+        assert_eq!(&dual.primary.settings.message_family, "messageFamily");
+
+        Ok(())
+    }
+}

From 64b199e177ab604a018fd584a09067196e00790b Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Mon, 28 Aug 2023 10:47:17 -0700
Subject: [PATCH 03/11] f add flag to write to secondary (default true)

This flag makes writing to the secondary an active feature. This is
required due to the potential for clients to be arbitrarily assigned to
the dual mode storage or the older single mode storage. This flag is set
to `true` as the default since we want both data stores active until we
have sufficient cross coverage.
---
 autopush-common/src/db/dual/mod.rs | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/autopush-common/src/db/dual/mod.rs b/autopush-common/src/db/dual/mod.rs
index 48afd85e2..3c272e22e 100644
--- a/autopush-common/src/db/dual/mod.rs
+++ b/autopush-common/src/db/dual/mod.rs
@@ -30,6 +30,7 @@ use crate::db::dynamodb::DdbClientImpl;
 pub struct DualClientImpl {
     primary: BigTableClientImpl,
     secondary: DdbClientImpl,
+    write_to_secondary: bool,
     _metrics: Arc<StatsdClient>,
 }
 
@@ -39,6 +40,12 @@ pub struct DualDbSettings {
     primary: DbSettings,
     #[serde(default)]
     secondary: DbSettings,
+    #[serde(default = "set_true")]
+    write_to_secondary: bool,
+}
+
+fn set_true() -> bool {
+    true
 }
 
 impl DualClientImpl {
@@ -63,6 +70,7 @@ impl DualClientImpl {
             primary,
             secondary,
             _metrics: metrics,
+            write_to_secondary: db_settings.write_to_secondary,
         })
     }
 }
@@ -70,10 +78,16 @@ impl DualClientImpl {
 #[async_trait]
 impl DbClient for DualClientImpl {
     async fn add_user(&self, user: &User) -> DbResult<()> {
+        if self.write_to_secondary {
+            let _ = self.secondary.add_user(user).await?;
+        }
         self.primary.add_user(user).await
     }
 
     async fn update_user(&self, user: &User) -> DbResult<()> {
+        if self.write_to_secondary {
+            let _ = self.secondary.update_user(user).await?;
+        }
         self.primary.update_user(user).await
     }
 
@@ -101,6 +115,9 @@ impl DbClient for DualClientImpl {
     }
 
     async fn add_channel(&self, uaid: &Uuid, channel_id: &Uuid) -> DbResult<()> {
+        if self.write_to_secondary {
+            let _ = self.secondary.add_channel(uaid, channel_id).await?;
+        }
         self.primary.add_channel(uaid, channel_id).await
     }
 
@@ -130,6 +147,9 @@ impl DbClient for DualClientImpl {
     }
 
     async fn save_message(&self, uaid: &Uuid, message: Notification) -> DbResult<()> {
+        if self.write_to_secondary {
+            let _ = self.secondary.save_message(uaid, message.clone()).await?;
+        }
         self.primary.save_message(uaid, message).await
     }
 
@@ -170,10 +190,16 @@ impl DbClient for DualClientImpl {
     }
 
     async fn save_messages(&self, uaid: &Uuid, messages: Vec<Notification>) -> DbResult<()> {
+        if self.write_to_secondary {
+            let _ = self.secondary.save_messages(uaid, messages.clone()).await?;
+        }
         self.primary.save_messages(uaid, messages).await
     }
 
     async fn increment_storage(&self, uaid: &Uuid, timestamp: u64) -> DbResult<()> {
+        if self.write_to_secondary {
+            let _ = self.secondary.increment_storage(uaid, timestamp).await?;
+        }
         self.primary.increment_storage(uaid, timestamp).await
     }
 
@@ -224,6 +250,7 @@ mod test {
                     "message_table": "test_message",
                 }).to_string(),
             },
+            "write_to_secondary": true
         })
         .to_string();
 

From bf78aeaaf9303cb88410f3af208c8ae9bfccdd4d Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Fri, 3 Nov 2023 14:52:46 -0700
Subject: [PATCH 04/11] f fix merge

---
 autopush-common/src/db/dual/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/autopush-common/src/db/dual/mod.rs b/autopush-common/src/db/dual/mod.rs
index 3c272e22e..3d25c23c5 100644
--- a/autopush-common/src/db/dual/mod.rs
+++ b/autopush-common/src/db/dual/mod.rs
@@ -84,7 +84,7 @@ impl DbClient for DualClientImpl {
         self.primary.add_user(user).await
     }
 
-    async fn update_user(&self, user: &User) -> DbResult<()> {
+    async fn update_user(&self, user: &User) -> DbResult<bool> {
         if self.write_to_secondary {
             let _ = self.secondary.update_user(user).await?;
         }

From 101cfaa506b00b629ba5cf43c7982e207b2d9836 Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Mon, 13 Nov 2023 15:35:37 -0800
Subject: [PATCH 05/11] f bump gcr-raw to 0.16

---
 Cargo.lock                 | 16 ++++++++--------
 autopush-common/Cargo.toml | 18 ++++++++++++------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index af5491221..95978286f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1023,9 +1023,9 @@ dependencies = [
 
 [[package]]
 name = "boringssl-src"
-version = "0.5.2+6195bf8"
+version = "0.6.0+e46383f"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ab565ccc5e276ea82a2013dd08bf2c999866b06daf1d4f30fee419c4aaec6d5"
+checksum = "5edec42197c014d84ea2396589f0da14b2257f63f319442b5e8475a077b90457"
 dependencies = [
  "cmake",
 ]
@@ -2061,9 +2061,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "google-cloud-rust-raw"
-version = "0.15.0"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbabcfb0bcc5e3222191c3f0fba962b0cbf4242d2effe2a865090eee492ffc9c"
+checksum = "05cfec9367aefac6e90f42db5dee124b64fa5bcf84adf4236a861453373d851c"
 dependencies = [
  "futures 0.3.28",
  "grpcio",
@@ -2072,9 +2072,9 @@ dependencies = [
 
 [[package]]
 name = "grpcio"
-version = "0.12.1"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "609832ca501baeb662dc81932fda9ed83f5d058f4b899a807ba222ce696f430a"
+checksum = "9e398946b5721d72478eb647260a1b7c1d5f70f0de35399846c3913bd369a33e"
 dependencies = [
  "futures-executor",
  "futures-util",
@@ -2087,9 +2087,9 @@ dependencies = [
 
 [[package]]
 name = "grpcio-sys"
-version = "0.12.1+1.46.5-patched"
+version = "0.13.0+1.56.2-patched"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf625d1803b6f44203f0428ddace847fb4994def5c803fc8a7a2f18fb3daec62"
+checksum = "b3dae9132320ae1b03ea55b5ddc88ca72a31fb85fa631a241a40157f5feffe43"
 dependencies = [
  "bindgen",
  "boringssl-src",
diff --git a/autopush-common/Cargo.toml b/autopush-common/Cargo.toml
index 8d7610d16..67aa96702 100644
--- a/autopush-common/Cargo.toml
+++ b/autopush-common/Cargo.toml
@@ -61,10 +61,15 @@ woothee = "0.13"
 
 # #[cfg(bigtable)] for this section.
 # the following three crates must match what is specifed in google-cloud-rust-raw's dependencies
-google-cloud-rust-raw = {version="0.15", default-features=false, features=["bigtable"]}
-#google-cloud-rust-raw = {path = "../../google-cloud-rust/googleapis-raw", version="0.15", default-features=false, features=["bigtable"]}
-grpcio = {version="=0.12.1", features=["openssl"]}
-protobuf = "=2.28.0"  # grpcio does not support protobuf 3+
+google-cloud-rust-raw = { version = "0.16", default-features = false, features = [
+    "bigtable",
+] }
+# google-cloud-rust-raw = { path = "../../google-cloud-rust/googleapis-raw", version = "0.16", default-features = false, features = ["bigtable"] }
+
+grpcio = { version = "=0.13", features = [
+    "openssl",
+] } # required by google-cloud-rust-raw
+protobuf = "=2.28.0" # grpcio does not support protobuf 3+
 
 [dev-dependencies]
 mockito = "0.31"
@@ -78,5 +83,6 @@ actix-rt = "2.8"
 default = ["dynamodb"]
 dynamodb = []
 bigtable = []
-emulator = ["bigtable"]  # used for testing big table, requires an external bigtable emulator running.
-
+emulator = [
+    "bigtable",
+] # used for testing big table, requires an external bigtable emulator running.

From d2b4bb6bb3e6639df345beba7c874458c244f8c0 Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Tue, 14 Nov 2023 09:03:52 -0800
Subject: [PATCH 06/11] f reduce the circleci jobs to fit memory

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0681d6f62..470cc2b27 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -113,7 +113,7 @@ jobs:
           # when doing discovery, we found that the docker image `meminfo` and `cpuinfo` often report
           # the machine level memory and CPU which are far higher than the memory allocated to the docker
           # instance. This may be causing rust to be overly greedy triggering the VM to OOM the process.)
-          command: cargo test --features=bigtable --jobs=8
+          command: cargo test --features=bigtable --jobs=2
       - run:
           name: Integration tests (Autopush Legacy)
           command: |

From 204a9d272afe81bf1282e14ae5132b87edcc7df7 Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Wed, 15 Nov 2023 09:16:53 -0800
Subject: [PATCH 07/11] f fix remove_node_id()

---
 autopush-common/src/db/dual/mod.rs | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/autopush-common/src/db/dual/mod.rs b/autopush-common/src/db/dual/mod.rs
index 3d25c23c5..42f1c082b 100644
--- a/autopush-common/src/db/dual/mod.rs
+++ b/autopush-common/src/db/dual/mod.rs
@@ -136,14 +136,21 @@ impl DbClient for DualClientImpl {
         Ok(presult | sresult)
     }
 
-    async fn remove_node_id(&self, uaid: &Uuid, node_id: &str, connected_at: u64) -> DbResult<()> {
-        let _ = self
+    async fn remove_node_id(
+        &self,
+        uaid: &Uuid,
+        node_id: &str,
+        connected_at: u64,
+    ) -> DbResult<bool> {
+        let p_resp = self
             .secondary
             .remove_node_id(uaid, node_id, connected_at)
-            .await;
-        self.primary
+            .await?;
+        let s_resp = self
+            .primary
             .remove_node_id(uaid, node_id, connected_at)
-            .await
+            .await?;
+        return Ok(p_resp || s_resp);
     }
 
     async fn save_message(&self, uaid: &Uuid, message: Notification) -> DbResult<()> {

From a21acc7b44c0f6d93180e029131633c2d79ed399 Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Fri, 17 Nov 2023 13:05:21 -0800
Subject: [PATCH 08/11] f post merge fixups

---
 autoendpoint/Cargo.toml      | 9 ++++++---
 autoendpoint/src/settings.rs | 8 ++++----
 autopush-common/Cargo.toml   | 3 ++-
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/autoendpoint/Cargo.toml b/autoendpoint/Cargo.toml
index 784c82782..d7cd43c8a 100644
--- a/autoendpoint/Cargo.toml
+++ b/autoendpoint/Cargo.toml
@@ -44,9 +44,12 @@ tokio.workspace = true
 url.workspace = true
 uuid.workspace = true
 
-a2 = {version = "0.8", git = "https://github.com/mozilla-services/a2", branch="master"}
+a2 = { version = "0.8", git = "https://github.com/mozilla-services/a2", branch = "master" }
 bytebuffer = "2.1"
-again = { version = "0.1.2", default-features = false, features = ["log", "rand"] }
+again = { version = "0.1.2", default-features = false, features = [
+    "log",
+    "rand",
+] }
 async-trait = "0.1"
 autopush_common = { path = "../autopush-common" }
 jsonwebtoken = "8.0"
@@ -67,5 +70,5 @@ tokio = { workspace = true, features = ["fs", "macros"] }
 default = ["dynamodb"]
 dynamodb = []
 emulator = ["bigtable"]
-dual = ["bigtable"]
+dual = ["bigtable", "dynamodb"]
 bigtable = ["autopush_common/bigtable"]
diff --git a/autoendpoint/src/settings.rs b/autoendpoint/src/settings.rs
index 412a7676e..8cd59d91a 100644
--- a/autoendpoint/src/settings.rs
+++ b/autoendpoint/src/settings.rs
@@ -56,10 +56,10 @@ impl Default for Settings {
             db_settings: "".to_owned(),
             router_table_name: "router".to_string(),
             message_table_name: "message".to_string(),
-            /// max data is a bit hard to figure out, due to encryption. Using something
-            /// like pywebpush, if you encode a block of 4096 bytes, you'll get a
-            /// 4216 byte data block. Since we're going to be receiving this, we have to
-            /// presume base64 encoding, so we can bump things up to 5630 bytes max.
+            // max data is a bit hard to figure out, due to encryption. Using something
+            // like pywebpush, if you encode a block of 4096 bytes, you'll get a
+            // 4216 byte data block. Since we're going to be receiving this, we have to
+            // presume base64 encoding, so we can bump things up to 5630 bytes max.
             max_data_bytes: 5630,
             crypto_keys: format!("[{}]", Fernet::generate_key()),
             auth_keys: r#"["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB="]"#.to_string(),
diff --git a/autopush-common/Cargo.toml b/autopush-common/Cargo.toml
index 0b8aff844..f693ad7b3 100644
--- a/autopush-common/Cargo.toml
+++ b/autopush-common/Cargo.toml
@@ -64,7 +64,7 @@ woothee = "0.13"
 # the following three crates must match what is specifed in google-cloud-rust-raw's dependencies
 google-cloud-rust-raw = { version = "0.16", features = [
     "bigtable",
-], optional = true } # features=["bigtable"]}
+], optional = true }
 grpcio = { version = "=0.13.0", features = ["openssl"], optional = true }
 protobuf = { version = "=2.28.0", optional = true } # grpcio does not support protobuf 3+
 
@@ -78,6 +78,7 @@ actix-rt = "2.8"
 # for testing:
 # default = ["emulator"]
 bigtable = ["dep:google-cloud-rust-raw", "dep:grpcio", "dep:protobuf"]
+dynamodb = []
 emulator = [
     "bigtable",
 ] # used for testing big table, requires an external bigtable emulator running.

From 577ca3fe01a3e9bffbc8efc1c4616b670ec8fd24 Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Fri, 17 Nov 2023 13:34:41 -0800
Subject: [PATCH 09/11] f post merge dependency fixes

---
 autoendpoint/Cargo.toml    |  2 +-
 autopush-common/Cargo.toml |  9 +++++----
 autopush/Cargo.toml        | 24 ++++++++++++------------
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/autoendpoint/Cargo.toml b/autoendpoint/Cargo.toml
index d7cd43c8a..eb11b5653 100644
--- a/autoendpoint/Cargo.toml
+++ b/autoendpoint/Cargo.toml
@@ -68,7 +68,7 @@ tokio = { workspace = true, features = ["fs", "macros"] }
 
 [features]
 default = ["dynamodb"]
-dynamodb = []
+dynamodb = ["autopush_common/dynamodb"]
 emulator = ["bigtable"]
 dual = ["bigtable", "dynamodb"]
 bigtable = ["autopush_common/bigtable"]
diff --git a/autopush-common/Cargo.toml b/autopush-common/Cargo.toml
index f693ad7b3..1069bf44e 100644
--- a/autopush-common/Cargo.toml
+++ b/autopush-common/Cargo.toml
@@ -28,9 +28,9 @@ openssl.workspace = true
 rand.workspace = true
 regex.workspace = true
 reqwest.workspace = true
-rusoto_core.workspace = true
-rusoto_credential.workspace = true
-rusoto_dynamodb.workspace = true
+rusoto_core = { workspace = true, optional = true }
+rusoto_credential = { workspace = true, optional = true }
+rusoto_dynamodb = { workspace = true, optional = true }
 sentry-backtrace.workspace = true
 sentry.workspace = true
 sentry-core.workspace = true
@@ -77,8 +77,9 @@ actix-rt = "2.8"
 [features]
 # for testing:
 # default = ["emulator"]
+default = ["dynamodb"]
 bigtable = ["dep:google-cloud-rust-raw", "dep:grpcio", "dep:protobuf"]
-dynamodb = []
+dynamodb = ["dep:rusoto_core", "dep:rusoto_credential", "dep:rusoto_dynamodb"]
 emulator = [
     "bigtable",
 ] # used for testing big table, requires an external bigtable emulator running.
diff --git a/autopush/Cargo.toml b/autopush/Cargo.toml
index 7e9b80545..d635fe476 100644
--- a/autopush/Cargo.toml
+++ b/autopush/Cargo.toml
@@ -36,25 +36,25 @@ slog-term.workspace = true
 tungstenite.workspace = true
 uuid.workspace = true
 
-autopush_common = { path = "../autopush-common" }
-bytes = "0.4" # XXX: pin to < 0.5 for hyper 0.12
+autopush_common = { path = "../autopush-common", features = ["dynamodb"] }
+bytes = "0.4"                                                              # XXX: pin to < 0.5 for hyper 0.12
 crossbeam-channel = "0.5"
 env_logger = "0.9"
 thiserror = "1.0"
-futures = "0.1.29" # XXX: pin to 0.1 until likely hyper 0.13
+futures = "0.1.29"                                                         # XXX: pin to 0.1 until likely hyper 0.13
 futures-backoff = "0.1.0"
-futures-locks = "0.5" # pin to 0.5 until futures update
-hyper = "^0.12.36" # pin to hyper 0.12 for now: 0.13 has many changes..
-reqwest = "0.9.24" # XXX: pin to < 0.10 until futures 0.3
-rusoto_core = "0.42.0"  # 0.46+ requires futures 0.3+
-rusoto_credential = "0.42.0"  # 0.46+ requires futures 0.3+
-rusoto_dynamodb = "0.42.0" # XXX: pin to 0.42 until futures 0.3
-serde_dynamodb = "0.4" # 0.7+ requires carg 0.46+
+futures-locks = "0.5"                                                      # pin to 0.5 until futures update
+hyper = "^0.12.36"                                                         # pin to hyper 0.12 for now: 0.13 has many changes..
+reqwest = "0.9.24"                                                         # XXX: pin to < 0.10 until futures 0.3
+rusoto_core = "0.42.0"                                                     # 0.46+ requires futures 0.3+
+rusoto_credential = "0.42.0"                                               # 0.46+ requires futures 0.3+
+rusoto_dynamodb = "0.42.0"                                                 # XXX: pin to 0.42 until futures 0.3
+serde_dynamodb = "0.4"                                                     # 0.7+ requires carg 0.46+
 signal-hook = "0.3"
 # state_machine_future = { version = "0.1.6", features = ["debug_code_generation"] }
 state_machine_future = "0.2.0"
 tokio-core = "0.1"
 tokio-io = "0.1.13"
-tokio-openssl = "0.3.0" # XXX: pin to < 0.4 until hyper 0.13
-tokio-tungstenite = { version = "0.9.0", default-features = false }  # 0.10+ requires tokio 0.3+
+tokio-openssl = "0.3.0"                                             # XXX: pin to < 0.4 until hyper 0.13
+tokio-tungstenite = { version = "0.9.0", default-features = false } # 0.10+ requires tokio 0.3+
 woothee = "0.13"

From f2753f75f249ed1da7626c473937476d22a9d99d Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Wed, 22 Nov 2023 10:59:18 -0800
Subject: [PATCH 10/11] f r's

---
 Cargo.toml                         | 4 +---
 autopush-common/src/db/dual/mod.rs | 9 +--------
 2 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 73388d641..56b934ce4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -78,9 +78,7 @@ rusoto_dynamodb = { version = "0.47", default-features = false, features = [
 sentry = { version = "0.31", features = [
   "debug-logs",
 ] } # Using debug-logs avoids https://github.com/getsentry/sentry-rust/issues/237
-sentry-core = { version = "0.31", default-features = false, features = [
-  "client",
-] }
+sentry-core = { version = "0.31" }
 sentry-actix = "0.31"
 sentry-backtrace = "0.31"
 serde = "1.0"
diff --git a/autopush-common/src/db/dual/mod.rs b/autopush-common/src/db/dual/mod.rs
index 42f1c082b..6ea3758a2 100644
--- a/autopush-common/src/db/dual/mod.rs
+++ b/autopush-common/src/db/dual/mod.rs
@@ -34,20 +34,13 @@ pub struct DualClientImpl {
     _metrics: Arc<StatsdClient>,
 }
 
-#[derive(Clone, Debug, Default, Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
 pub struct DualDbSettings {
-    #[serde(default)]
     primary: DbSettings,
-    #[serde(default)]
     secondary: DbSettings,
-    #[serde(default = "set_true")]
     write_to_secondary: bool,
 }
 
-fn set_true() -> bool {
-    true
-}
-
 impl DualClientImpl {
     pub fn new(metrics: Arc<StatsdClient>, settings: &DbSettings) -> DbResult<Self> {
         // Not really sure we need the dsn here.

From a4829cf9ea80ced059aca8a0616e24136018e838 Mon Sep 17 00:00:00 2001
From: jrconlin <jr+git@mozilla.com>
Date: Wed, 29 Nov 2023 13:13:46 -0800
Subject: [PATCH 11/11] f cleanup (why didn't clippy catch that?)

---
 autopush-common/src/db/dual/mod.rs | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/autopush-common/src/db/dual/mod.rs b/autopush-common/src/db/dual/mod.rs
index 6ea3758a2..79213ac08 100644
--- a/autopush-common/src/db/dual/mod.rs
+++ b/autopush-common/src/db/dual/mod.rs
@@ -16,22 +16,20 @@ use serde_json::from_str;
 use uuid::Uuid;
 
 use crate::db::{
+    bigtable::BigTableClientImpl,
     client::{DbClient, FetchMessageResponse},
+    dynamodb::DdbClientImpl,
     error::{DbError, DbResult},
     DbSettings, Notification, User,
 };
 
 use super::StorageType;
 
-use crate::db::bigtable::BigTableClientImpl;
-use crate::db::dynamodb::DdbClientImpl;
-
 #[derive(Clone)]
 pub struct DualClientImpl {
     primary: BigTableClientImpl,
     secondary: DdbClientImpl,
     write_to_secondary: bool,
-    _metrics: Arc<StatsdClient>,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -43,7 +41,6 @@ pub struct DualDbSettings {
 
 impl DualClientImpl {
     pub fn new(metrics: Arc<StatsdClient>, settings: &DbSettings) -> DbResult<Self> {
-        // Not really sure we need the dsn here.
         let db_settings: DualDbSettings = from_str(&settings.db_settings).map_err(|e| {
             DbError::General(format!("Could not parse DualDBSettings string {:?}", e))
         })?;
@@ -62,7 +59,6 @@ impl DualClientImpl {
         Ok(Self {
             primary,
             secondary,
-            _metrics: metrics,
             write_to_secondary: db_settings.write_to_secondary,
         })
     }
@@ -135,15 +131,15 @@ impl DbClient for DualClientImpl {
         node_id: &str,
         connected_at: u64,
     ) -> DbResult<bool> {
-        let p_resp = self
+        let presult = self
             .secondary
             .remove_node_id(uaid, node_id, connected_at)
             .await?;
-        let s_resp = self
+        let sresult = self
             .primary
             .remove_node_id(uaid, node_id, connected_at)
             .await?;
-        return Ok(p_resp || s_resp);
+        Ok(presult || sresult)
     }
 
     async fn save_message(&self, uaid: &Uuid, message: Notification) -> DbResult<()> {
@@ -167,7 +163,7 @@ impl DbClient for DualClientImpl {
         if result.messages.is_empty() {
             return self.secondary.fetch_topic_messages(uaid, limit).await;
         }
-        return Ok(result);
+        Ok(result)
     }
 
     async fn fetch_timestamp_messages(
@@ -186,7 +182,7 @@ impl DbClient for DualClientImpl {
                 .fetch_timestamp_messages(uaid, timestamp, limit)
                 .await;
         }
-        return Ok(result);
+        Ok(result)
     }
 
     async fn save_messages(&self, uaid: &Uuid, messages: Vec<Notification>) -> DbResult<()> {