From 6d799304596570d01b40e4f52515f385385d7255 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Fri, 10 Jul 2020 13:25:22 -0400 Subject: [PATCH 1/5] Copy and upgrade parts of DynamoStorage into autoendpoint Autoendpoint no longer uses any Tokio 0.1 code (and consequently doesn't crash). Some of the autopush_common macros are exported so they can be used in the autoendpoint database client. The autoendpoint database client returns an option for get_user instead of an error if it doesn't exist. --- Cargo.lock | 246 ++++++++++++++++++++ autoendpoint/Cargo.toml | 4 + autoendpoint/src/db/client.rs | 184 +++++++++++++++ autoendpoint/src/db/error.rs | 24 ++ autoendpoint/src/db/mod.rs | 7 + autoendpoint/src/db/retry.rs | 34 +++ autoendpoint/src/error.rs | 3 +- autoendpoint/src/extractors/subscription.rs | 9 +- autoendpoint/src/extractors/user.rs | 27 +-- autoendpoint/src/main.rs | 1 + autoendpoint/src/routers/mod.rs | 3 +- autoendpoint/src/routers/webpush.rs | 24 +- autoendpoint/src/server.rs | 15 +- autopush-common/src/db/macros.rs | 11 +- 14 files changed, 539 insertions(+), 53 deletions(-) create mode 100644 autoendpoint/src/db/client.rs create mode 100644 autoendpoint/src/db/error.rs create mode 100644 autoendpoint/src/db/mod.rs create mode 100644 autoendpoint/src/db/retry.rs diff --git a/Cargo.lock b/Cargo.lock index aa2f637b4..37667f06d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -261,6 +261,14 @@ name = "adler32" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "again" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wasm-timer 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "aho-corasick" version = "0.7.9" @@ -340,6 +348,7 @@ dependencies = [ "actix-http 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "actix-rt 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "actix-web 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "again 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "async-trait 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", "autopush_common 1.0.0", "backtrace 0.3.44 (registry+https://github.com/rust-lang/crates.io-index)", @@ -356,8 +365,11 @@ dependencies = [ "openssl 0.10.29 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_core 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_dynamodb 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", "sentry 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_dynamodb 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "slog-async 2.5.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -503,6 +515,11 @@ dependencies = [ "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "base-x" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "base64" version = "0.10.1" @@ -996,6 +1013,11 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "docopt" version = "1.1.0" @@ -2532,6 +2554,34 @@ dependencies = [ "xml-rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rusoto_core" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "async-trait 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-tls 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "md5 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_credential 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_signature 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "xml-rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rusoto_credential" version = "0.42.0" @@ -2551,6 +2601,25 @@ dependencies = [ "tokio-timer 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rusoto_credential" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "async-trait 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", + "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", + "shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rusoto_dynamodb" version = "0.42.0" @@ -2564,6 +2633,19 @@ dependencies = [ "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rusoto_dynamodb" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "async-trait 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_core 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rusoto_signature" version = "0.42.0" @@ -2587,6 +2669,30 @@ dependencies = [ "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rusoto_signature" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "md5 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_credential 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rust-argon2" version = "0.7.0" @@ -2730,6 +2836,11 @@ name = "semver-parser" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "send_wrapper" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "sentry" version = "0.18.1" @@ -2812,6 +2923,16 @@ dependencies = [ "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde_dynamodb" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_dynamodb 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde_json" version = "1.0.53" @@ -3032,6 +3153,11 @@ name = "spin" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "standback" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "state_machine_future" version = "0.2.0" @@ -3047,6 +3173,51 @@ name = "static_assertions" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "discard 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "stdweb-derive 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "stdweb-internal-macros 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "stdweb-internal-runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base-x 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", + "sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "string" version = "0.2.1" @@ -3185,6 +3356,41 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "time" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", + "standback 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "stdweb 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", + "time-macros 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)", + "time-macros-impl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.16 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "standback 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tokio" version = "0.1.22" @@ -3221,6 +3427,7 @@ dependencies = [ "libc 0.2.71 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", + "mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "pin-project-lite 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3827,6 +4034,21 @@ name = "wasm-bindgen-shared" version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "wasm-timer" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "send_wrapper 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-futures 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "web-sys" version = "0.3.41" @@ -3963,6 +4185,11 @@ dependencies = [ "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "zeroize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [metadata] "checksum actix-codec 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "09e55f0a5c2ca15795035d90c46bd0e73a5123b72f68f12596d6ba5282051380" "checksum actix-connect 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c95cc9569221e9802bf4c377f6c18b90ef10227d787611decf79fd47d2a8e76c" @@ -3980,6 +4207,7 @@ dependencies = [ "checksum actix-web 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3158e822461040822f0dbf1735b9c2ce1f95f93b651d7a7aded00b1efbb1f635" "checksum actix-web-codegen 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a71bf475cbe07281d0b3696abb48212db118e7e23219f13596ce865235ff5766" "checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" +"checksum again 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "05802a5ad4d172eaf796f7047b42d0af9db513585d16d4169660a21613d34b93" "checksum aho-corasick 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)" = "d5e63fd144e18ba274ae7095c0197a870a7b9468abc801dd62f190d80817d2ec" "checksum arc-swap 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff" "checksum arrayref 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" @@ -3993,6 +4221,7 @@ dependencies = [ "checksum awc 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d7601d4d1d7ef2335d6597a41b5fe069f6ab799b85f53565ab390e7b7065aac5" "checksum backtrace 0.3.44 (registry+https://github.com/rust-lang/crates.io-index)" = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536" "checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" +"checksum base-x 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" "checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" "checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" "checksum base64 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "53d1ccbaf7d9ec9537465a97bf19edc1a4e158ecb49fc16178202238c569cc42" @@ -4049,6 +4278,7 @@ dependencies = [ "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" "checksum dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" "checksum dirs-sys 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "afa0b23de8fd801745c471deffa6e12d248f962c9fd4b4c33787b055599bde7b" +"checksum discard 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" "checksum docopt 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f525a586d310c87df72ebcd98009e57f1cc030c8c268305287a476beb653969" "checksum dtoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" @@ -4213,9 +4443,13 @@ dependencies = [ "checksum resolv-conf 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "11834e137f3b14e309437a8276714eed3a80d1ef894869e510f2c0c0b98b9f4a" "checksum ring 0.16.15 (registry+https://github.com/rust-lang/crates.io-index)" = "952cd6b98c85bbc30efa1ba5783b8abf12fec8b3287ffa52605b9432313e34e4" "checksum rusoto_core 0.42.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f1d1ecfe8dac29878a713fbc4c36b0a84a48f7a6883541841cdff9fdd2ba7dfb" +"checksum rusoto_core 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)" = "841ca8f73e7498ba39146ab43acea906bbbb807d92ec0b7ea4b6293d2621f80d" "checksum rusoto_credential 0.42.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8632e41d289db90dd40d0389c71a23c5489e3afd448424226529113102e2a002" +"checksum rusoto_credential 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60669ddc1bdbb83ce225593649d36b4c5f6bf9db47cc1ab3e81281abffc853f4" "checksum rusoto_dynamodb 0.42.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f6cfb692b9300c14d0d7c2607b2dcb9da8afca4239f3ca4e9ec48f47696ac9d" +"checksum rusoto_dynamodb 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a485bf81a63fd92a4e011b76daed2731b363a6f2b6279b8b26b389699bcf1525" "checksum rusoto_signature 0.42.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7063a70614eb4b36f49bcf4f6f6bb30cc765e3072b317d6afdfe51e7a9f482d1" +"checksum rusoto_signature 0.44.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9eddff187ac18c5a91d9ccda9353f30cf531620dce437c4db661dfe2e23b2029" "checksum rust-argon2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" "checksum rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" @@ -4234,6 +4468,7 @@ dependencies = [ "checksum security-framework-sys 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum send_wrapper 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a0eddf2e8f50ced781f288c19f18621fa72a3779e3cb58dbf23b07469b0abeb4" "checksum sentry 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8b01b723fc1b0a0f9394ca1a8451daec6e20206d47f96c3dceea7fd11ec9eec0" "checksum sentry-types 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "12ec406c11c060c8a7d5d67fc6f4beb2888338dcb12b9af409451995f124749d" "checksum serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" @@ -4241,6 +4476,7 @@ dependencies = [ "checksum serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8" "checksum serde_derive 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)" = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" "checksum serde_dynamodb 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5a2f694b5265a3612a5f51ce734e71b1865850a13b9390fea3da60a65dba6218" +"checksum serde_dynamodb 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "21ceb0b8298ba5707f1c2573109d16ca31434b4449ab07a6e064d9f60fa20d7a" "checksum serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)" = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2" "checksum serde_test 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5" "checksum serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a" @@ -4265,8 +4501,13 @@ dependencies = [ "checksum smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" "checksum socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "e8b74de517221a2cb01a53349cf54182acdc31a074727d3079068448c0676d85" "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +"checksum standback 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "47e4b8c631c998468961a9ea159f064c5c8499b95b5e4a34b77849d45949d540" "checksum state_machine_future 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "530e1d624baae485bce12e6647acb76aafa253346ee8a16751974eed5a24b13d" "checksum static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3" +"checksum stdweb 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +"checksum stdweb-derive 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +"checksum stdweb-internal-macros 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +"checksum stdweb-internal-runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" "checksum strsim 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" "checksum subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" @@ -4283,6 +4524,9 @@ dependencies = [ "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum threadpool 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +"checksum time 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "3a51cadc5b1eec673a685ff7c33192ff7b7603d0b75446fb354939ee615acb15" +"checksum time-macros 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9ae9b6e9f095bc105e183e3cd493d72579be3181ad4004fceb01adbe9eecab2d" +"checksum time-macros-impl 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" "checksum tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" "checksum tokio 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "b34bee1facdc352fba10c9c58b654e6ecb6a2250167772bf86071f7c5f2f5061" "checksum tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" @@ -4343,6 +4587,7 @@ dependencies = [ "checksum wasm-bindgen-macro 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "3fcfd5ef6eec85623b4c6e844293d4516470d8f19cd72d0d12246017eb9060b8" "checksum wasm-bindgen-macro-support 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "9adff9ee0e94b926ca81b57f57f86d5545cdcb1d259e21ec9bdd95b901754c75" "checksum wasm-bindgen-shared 0.2.64 (registry+https://github.com/rust-lang/crates.io-index)" = "7f7b90ea6c632dd06fd765d44542e234d5e63d9bb917ecd64d79778a13bd79ae" +"checksum wasm-timer 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "324c5e65a08699c9c4334ba136597ab22b85dccd4b65dd1e36ccf8f723a95b54" "checksum web-sys 0.3.41 (registry+https://github.com/rust-lang/crates.io-index)" = "863539788676619aac1a23e2df3655e96b32b0e05eb72ca34ba045ad573c625d" "checksum webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae" "checksum widestring 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "effc0e4ff8085673ea7b9b2e3c73f6bd4d118810c9009ed8f1e16bd96c331db6" @@ -4360,3 +4605,4 @@ dependencies = [ "checksum xml-rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "541b12c998c5b56aa2b4e6f18f03664eef9a4fd0a246a55594efae6cc2d964b5" "checksum yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d" "checksum yup-oauth2 4.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "749192b9464694a95dbaf0586e845c835b315e38d491aa2766a8477aaadb48ec" +"checksum zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" diff --git a/autoendpoint/Cargo.toml b/autoendpoint/Cargo.toml index 2d4872958..ae973dc35 100644 --- a/autoendpoint/Cargo.toml +++ b/autoendpoint/Cargo.toml @@ -9,6 +9,7 @@ actix-http = "1.0" actix-web = "2.0" actix-rt = "1.0" actix-cors = "0.2.0" +again = { version = "0.1.2", default-features = false } async-trait = "0.1.36" autopush_common = { path = "../autopush-common" } backtrace = "0.3" @@ -24,8 +25,11 @@ lazy_static = "1.4.0" openssl = "0.10" regex = "1.3" reqwest = "0.10.6" +rusoto_core = "0.44.0" +rusoto_dynamodb = "0.44.0" sentry = { version = "0.18", features = ["with_curl_transport"] } serde = { version = "1.0", features = ["derive"] } +serde_dynamodb = "0.5.1" serde_json = "1.0" slog = { version = "2.5", features = ["max_level_trace", "release_max_level_error", "dynamic-keys"] } slog-async = "2.4" diff --git a/autoendpoint/src/db/client.rs b/autoendpoint/src/db/client.rs new file mode 100644 index 000000000..f00b31f9f --- /dev/null +++ b/autoendpoint/src/db/client.rs @@ -0,0 +1,184 @@ +use crate::db::error::{DbError, DbResult}; +use crate::db::retry::{ + retry_policy, retryable_delete_error, retryable_getitem_error, retryable_putitem_error, + retryable_updateitem_error, +}; +use autopush_common::db::{DynamoDbNotification, DynamoDbUser}; +use autopush_common::notification::Notification; +use autopush_common::{ddb_item, hashmap, val}; +use cadence::StatsdClient; +use rusoto_core::credential::StaticProvider; +use rusoto_core::{HttpClient, Region}; +use rusoto_dynamodb::{ + AttributeValue, DeleteItemInput, DynamoDb, DynamoDbClient, GetItemInput, PutItemInput, + UpdateItemInput, +}; +use std::collections::HashSet; +use std::env; +use uuid::Uuid; + +/// Provides high-level operations over the DynamoDB database +#[derive(Clone)] +pub struct DbClient { + ddb: DynamoDbClient, + metrics: StatsdClient, + router_table: String, + pub message_table: String, +} + +impl DbClient { + pub fn new( + metrics: StatsdClient, + router_table: String, + message_table: String, + ) -> DbResult { + let ddb = if let Ok(endpoint) = env::var("AWS_LOCAL_DYNAMODB") { + DynamoDbClient::new_with( + HttpClient::new().expect("TLS initialization error"), + StaticProvider::new_minimal("BogusKey".to_string(), "BogusKey".to_string()), + Region::Custom { + name: "us-east-1".to_string(), + endpoint, + }, + ) + } else { + DynamoDbClient::new(Region::default()) + }; + + Ok(Self { + ddb, + metrics, + router_table, + message_table, + }) + } + + /// Read a user from the database + pub async fn get_user(&self, uaid: Uuid) -> DbResult> { + let input = GetItemInput { + table_name: self.router_table.clone(), + consistent_read: Some(true), + key: ddb_item! { uaid: s => uaid.to_simple().to_string() }, + ..Default::default() + }; + + retry_policy() + .retry_if( + || self.ddb.get_item(input.clone()), + retryable_getitem_error(self.metrics.clone()), + ) + .await? + .item + .map(serde_dynamodb::from_hashmap) + .transpose() + .map_err(DbError::from) + } + + /// Delete a user from the router table + pub async fn drop_user(&self, uaid: Uuid) -> DbResult<()> { + let input = DeleteItemInput { + table_name: self.router_table.clone(), + key: ddb_item! { uaid: s => uaid.to_simple().to_string() }, + ..Default::default() + }; + + retry_policy() + .retry_if( + || self.ddb.delete_item(input.clone()), + retryable_delete_error(self.metrics.clone()), + ) + .await?; + Ok(()) + } + + /// Get the set of channel IDs for a user + pub async fn get_user_channels(&self, uaid: Uuid) -> DbResult> { + // Channel IDs are stored in a special row in the message table, where + // chidmessageid = " " + let input = GetItemInput { + table_name: self.message_table.clone(), + consistent_read: Some(true), + key: ddb_item! { + uaid: s => uaid.to_simple().to_string(), + chidmessageid: s => " ".to_string() + }, + ..Default::default() + }; + + let output = retry_policy() + .retry_if( + || self.ddb.get_item(input.clone()), + retryable_getitem_error(self.metrics.clone()), + ) + .await?; + + // The channel IDs are in the notification's `chids` field + let channels = output + .item + // Deserialize the notification + .map(serde_dynamodb::from_hashmap::) + .transpose()? + // Extract the channel IDs + .and_then(|n| n.chids) + .unwrap_or_default(); + + // Convert the IDs from String to Uuid + let channels = channels + .into_iter() + .map(|s| Uuid::parse_str(&s)) + .filter_map(Result::ok) + .collect(); + + Ok(channels) + } + + /// Remove the node ID from a user in the router table. + /// The node ID will only be cleared if `connected_at` matches up with the + /// item's `connected_at`. + pub async fn remove_node_id( + &self, + uaid: Uuid, + node_id: String, + connected_at: u64, + ) -> DbResult<()> { + let update_item = UpdateItemInput { + key: ddb_item! { uaid: s => uaid.to_simple().to_string() }, + update_expression: Some("REMOVE node_id".to_string()), + condition_expression: Some("(node_id = :node) and (connected_at = :conn)".to_string()), + expression_attribute_values: Some(hashmap! { + ":node".to_string() => val!(S => node_id), + ":conn".to_string() => val!(N => connected_at.to_string()) + }), + table_name: self.router_table.clone(), + ..Default::default() + }; + + retry_policy() + .retry_if( + || self.ddb.update_item(update_item.clone()), + retryable_updateitem_error(self.metrics.clone()), + ) + .await?; + + Ok(()) + } + + /// Store a single message + pub async fn store_message(&self, uaid: Uuid, message: Notification) -> DbResult<()> { + let put_item = PutItemInput { + item: serde_dynamodb::to_hashmap(&DynamoDbNotification::from_notif(&uaid, message)) + .unwrap(), + table_name: self.message_table.clone(), + ..Default::default() + }; + + retry_policy() + .retry_if( + || self.ddb.put_item(put_item.clone()), + retryable_putitem_error(self.metrics.clone()), + ) + .await?; + + Ok(()) + } +} diff --git a/autoendpoint/src/db/error.rs b/autoendpoint/src/db/error.rs new file mode 100644 index 000000000..05d43084c --- /dev/null +++ b/autoendpoint/src/db/error.rs @@ -0,0 +1,24 @@ +use thiserror::Error; + +use rusoto_core::RusotoError; +use rusoto_dynamodb::{DeleteItemError, GetItemError, PutItemError, UpdateItemError}; + +pub type DbResult = Result; + +#[derive(Debug, Error)] +pub enum DbError { + #[error(transparent)] + GetItem(#[from] RusotoError), + + #[error(transparent)] + UpdateItem(#[from] RusotoError), + + #[error(transparent)] + PutItem(#[from] RusotoError), + + #[error(transparent)] + DeleteItem(#[from] RusotoError), + + #[error("Error while deserializing database response: {0}")] + Deserialize(#[from] serde_dynamodb::Error), +} diff --git a/autoendpoint/src/db/mod.rs b/autoendpoint/src/db/mod.rs new file mode 100644 index 000000000..48ede147a --- /dev/null +++ b/autoendpoint/src/db/mod.rs @@ -0,0 +1,7 @@ +//! This DynamoDB client is a selectively upgraded version of `DynamoStorage` in `autopush_common`. +//! Due to #172, autoendpoint cannot use any Tokio 0.1 code, so for now we have to copy and update +//! pieces of `DynamoStorage` as needed. + +pub mod client; +pub mod error; +mod retry; diff --git a/autoendpoint/src/db/retry.rs b/autoendpoint/src/db/retry.rs new file mode 100644 index 000000000..974d755a7 --- /dev/null +++ b/autoendpoint/src/db/retry.rs @@ -0,0 +1,34 @@ +use again::RetryPolicy; +use cadence::{Counted, StatsdClient}; +use rusoto_core::RusotoError; +use rusoto_dynamodb::{DeleteItemError, GetItemError, PutItemError, UpdateItemError}; +use std::time::Duration; + +/// Create a retry function for the given error +macro_rules! retryable_error { + ($name:ident, $error:tt, $error_tag:expr) => { + pub fn $name(metrics: StatsdClient) -> impl Fn(&RusotoError<$error>) -> bool { + move |err| match err { + RusotoError::Service($error::InternalServerError(_)) + | RusotoError::Service($error::ProvisionedThroughputExceeded(_)) => { + metrics + .incr_with_tags("database.retry") + .with_tag("error", $error_tag) + .send(); + true + } + _ => false, + } + } + }; +} + +retryable_error!(retryable_getitem_error, GetItemError, "get_item"); +retryable_error!(retryable_updateitem_error, UpdateItemError, "update_item"); +retryable_error!(retryable_putitem_error, PutItemError, "put_item"); +retryable_error!(retryable_delete_error, DeleteItemError, "delete_item"); + +/// Build an exponential retry policy +pub fn retry_policy() -> RetryPolicy { + RetryPolicy::exponential(Duration::from_millis(100)) +} diff --git a/autoendpoint/src/error.rs b/autoendpoint/src/error.rs index 2e248d61c..fd0ad1c67 100644 --- a/autoendpoint/src/error.rs +++ b/autoendpoint/src/error.rs @@ -1,5 +1,6 @@ //! Error types and transformations +use crate::db::error::DbError; use crate::headers::vapid::VapidError; use crate::routers::RouterError; use actix_web::{ @@ -71,7 +72,7 @@ pub enum ApiErrorKind { TokenHashValidation(#[source] openssl::error::ErrorStack), #[error("Database error: {0}")] - Database(#[source] autopush_common::errors::Error), + Database(#[from] DbError), #[error("Invalid token")] InvalidToken, diff --git a/autoendpoint/src/extractors/subscription.rs b/autoendpoint/src/extractors/subscription.rs index b09726f4d..189b78060 100644 --- a/autoendpoint/src/extractors/subscription.rs +++ b/autoendpoint/src/extractors/subscription.rs @@ -11,7 +11,6 @@ use actix_web::{FromRequest, HttpRequest}; use autopush_common::db::DynamoDbUser; use autopush_common::util::sec_since_epoch; use cadence::{Counted, StatsdClient}; -use futures::compat::Future01CompatExt; use futures::future::LocalBoxFuture; use futures::FutureExt; use jsonwebtoken::{Algorithm, DecodingKey, Validation}; @@ -65,11 +64,9 @@ impl FromRequest for Subscription { let channel_id = Uuid::from_slice(&token[16..32]).unwrap(); let user = state .ddb - .get_user(&uaid) - .compat() - .await - .map_err(ApiErrorKind::Database)? - .ok_or(ApiErrorKind::NoUser)?; + .get_user(uaid) + .await? + .ok_or(ApiErrorKind::NoSubscription)?; let router_type = validate_user(&user, &channel_id, &state).await?; // Validate the VAPID JWT token and record the version diff --git a/autoendpoint/src/extractors/user.rs b/autoendpoint/src/extractors/user.rs index c35a9af8d..56e57677c 100644 --- a/autoendpoint/src/extractors/user.rs +++ b/autoendpoint/src/extractors/user.rs @@ -1,11 +1,11 @@ //! User validations +use crate::db::client::DbClient; use crate::error::{ApiErrorKind, ApiResult}; use crate::extractors::routers::RouterType; use crate::server::ServerState; -use autopush_common::db::{DynamoDbUser, DynamoStorage}; +use autopush_common::db::DynamoDbUser; use cadence::{Counted, StatsdClient}; -use futures::compat::Future01CompatExt; use uuid::Uuid; /// Perform some validations on the user, including: @@ -23,7 +23,7 @@ pub async fn validate_user( Ok(router_type) => router_type, Err(_) => { debug!("Unknown router type, dropping user"; "user" => ?user); - drop_user(&user.uaid, &state.ddb, &state.metrics).await?; + drop_user(user.uaid, &state.ddb, &state.metrics).await?; return Err(ApiErrorKind::NoSubscription.into()); } }; @@ -39,7 +39,7 @@ pub async fn validate_user( async fn validate_webpush_user( user: &DynamoDbUser, channel_id: &Uuid, - ddb: &DynamoStorage, + ddb: &DbClient, metrics: &StatsdClient, ) -> ApiResult<()> { // Make sure the user is active (has a valid message table) @@ -47,23 +47,19 @@ async fn validate_webpush_user( Some(table) => table, None => { debug!("Missing `current_month` value, dropping user"; "user" => ?user); - drop_user(&user.uaid, ddb, metrics).await?; + drop_user(user.uaid, ddb, metrics).await?; return Err(ApiErrorKind::NoSubscription.into()); } }; - if !ddb.message_table_names.contains(message_table) { + if ddb.message_table.as_str() != message_table { debug!("User is inactive, dropping user"; "user" => ?user); - drop_user(&user.uaid, ddb, metrics).await?; + drop_user(user.uaid, ddb, metrics).await?; return Err(ApiErrorKind::NoSubscription.into()); } // Make sure the subscription channel exists - let channel_ids = ddb - .get_user_channels(&user.uaid, message_table) - .compat() - .await - .map_err(ApiErrorKind::Database)?; + let channel_ids = ddb.get_user_channels(user.uaid).await?; if !channel_ids.contains(channel_id) { return Err(ApiErrorKind::NoSubscription.into()); @@ -73,16 +69,13 @@ async fn validate_webpush_user( } /// Drop a user and increment associated metric -async fn drop_user(uaid: &Uuid, ddb: &DynamoStorage, metrics: &StatsdClient) -> ApiResult<()> { +async fn drop_user(uaid: Uuid, ddb: &DbClient, metrics: &StatsdClient) -> ApiResult<()> { metrics .incr_with_tags("updates.drop_user") .with_tag("errno", "102") .send(); - ddb.drop_uaid(uaid) - .compat() - .await - .map_err(ApiErrorKind::Database)?; + ddb.drop_user(uaid).await?; Ok(()) } diff --git a/autoendpoint/src/main.rs b/autoendpoint/src/main.rs index b76a0ee80..050bcc4a8 100644 --- a/autoendpoint/src/main.rs +++ b/autoendpoint/src/main.rs @@ -3,6 +3,7 @@ #[macro_use] extern crate slog_scope; +mod db; mod error; mod extractors; mod headers; diff --git a/autoendpoint/src/routers/mod.rs b/autoendpoint/src/routers/mod.rs index 2c1ccd8ef..329c7a708 100644 --- a/autoendpoint/src/routers/mod.rs +++ b/autoendpoint/src/routers/mod.rs @@ -1,5 +1,6 @@ //! Routers route notifications to user agents +use crate::db::error::DbError; use crate::error::ApiResult; use crate::extractors::notification::Notification; use crate::routers::fcm::error::FcmError; @@ -60,7 +61,7 @@ pub enum RouterError { Fcm(#[from] FcmError), #[error("Database error while saving notification")] - SaveDb(#[source] autopush_common::errors::Error), + SaveDb(#[source] DbError), #[error("User was deleted during routing")] UserWasDeleted, diff --git a/autoendpoint/src/routers/webpush.rs b/autoendpoint/src/routers/webpush.rs index 4301061d8..62806f214 100644 --- a/autoendpoint/src/routers/webpush.rs +++ b/autoendpoint/src/routers/webpush.rs @@ -1,10 +1,10 @@ -use crate::error::{ApiErrorKind, ApiResult}; +use crate::db::client::DbClient; +use crate::error::{ApiError, ApiErrorKind, ApiResult}; use crate::extractors::notification::Notification; use crate::routers::{Router, RouterError, RouterResponse}; use async_trait::async_trait; -use autopush_common::db::{DynamoDbUser, DynamoStorage}; +use autopush_common::db::DynamoDbUser; use cadence::{Counted, StatsdClient}; -use futures::compat::Future01CompatExt; use reqwest::{Response, StatusCode}; use std::collections::HashMap; use url::Url; @@ -16,7 +16,7 @@ use uuid::Uuid; /// server is located via the database routing table. If the server is busy or /// not available, the notification is stored in the database. pub struct WebPushRouter { - pub ddb: DynamoStorage, + pub ddb: DbClient, pub metrics: StatsdClient, pub http: reqwest::Client, pub endpoint_url: Url, @@ -66,7 +66,7 @@ impl Router for WebPushRouter { // Retrieve the user data again, they may have reconnected or the node // is no longer busy. trace!("Re-fetching user to trigger notification check"); - let user = match self.ddb.get_user(&user.uaid).compat().await { + let user = match self.ddb.get_user(user.uaid).await { Ok(Some(user)) => user, Ok(None) => { trace!("No user found, must have been deleted"); @@ -140,16 +140,9 @@ impl WebPushRouter { async fn store_notification(&self, notification: &Notification) -> ApiResult<()> { self.ddb .store_message( - ¬ification.subscription.user.uaid, - notification - .subscription - .user - .current_month - .clone() - .unwrap_or_else(|| self.ddb.current_message_month.clone()), + notification.subscription.user.uaid, notification.clone().into(), ) - .compat() .await .map_err(|e| ApiErrorKind::Router(RouterError::SaveDb(e)).into()) } @@ -160,10 +153,9 @@ impl WebPushRouter { self.metrics.incr("updates.client.host_gone").ok(); self.ddb - .remove_node_id(&user.uaid, node_id, user.connected_at) - .compat() + .remove_node_id(user.uaid, node_id, user.connected_at) .await - .map_err(|e| ApiErrorKind::Database(e).into()) + .map_err(ApiError::from) } /// Update metrics and create a response for when a notification has been directly forwarded to diff --git a/autoendpoint/src/server.rs b/autoendpoint/src/server.rs index 0d9b7690e..a23fb6a7e 100644 --- a/autoendpoint/src/server.rs +++ b/autoendpoint/src/server.rs @@ -1,6 +1,7 @@ //! Main application server -use crate::error::{ApiError, ApiErrorKind, ApiResult}; +use crate::db::client::DbClient; +use crate::error::{ApiError, ApiResult}; use crate::metrics; use crate::routers::fcm::router::FcmRouter; use crate::routes::health::{health_route, lb_heartbeat_route, status_route, version_route}; @@ -10,7 +11,6 @@ use actix_cors::Cors; use actix_web::{ dev, http::StatusCode, middleware::errhandlers::ErrorHandlers, web, App, HttpServer, }; -use autopush_common::db::DynamoStorage; use cadence::StatsdClient; use fernet::MultiFernet; use std::sync::Arc; @@ -21,7 +21,7 @@ pub struct ServerState { pub metrics: StatsdClient, pub settings: Settings, pub fernet: Arc, - pub ddb: DynamoStorage, + pub ddb: DbClient, pub http: reqwest::Client, pub fcm_router: Arc, } @@ -33,12 +33,11 @@ impl Server { let metrics = metrics::metrics_from_opts(&settings)?; let bind_address = format!("{}:{}", settings.host, settings.port); let fernet = Arc::new(settings.make_fernet()); - let ddb = DynamoStorage::from_opts( - &settings.message_table_name, - &settings.router_table_name, + let ddb = DbClient::new( metrics.clone(), - ) - .map_err(ApiErrorKind::Database)?; + settings.router_table_name.clone(), + settings.message_table_name.clone(), + )?; let http = reqwest::Client::new(); let fcm_router = Arc::new( FcmRouter::new( diff --git a/autopush-common/src/db/macros.rs b/autopush-common/src/db/macros.rs index 3e716ddf4..2e514ad91 100644 --- a/autopush-common/src/db/macros.rs +++ b/autopush-common/src/db/macros.rs @@ -33,6 +33,7 @@ macro_rules! key_schema { } } +#[macro_export] macro_rules! val { (B => $val:expr) => {{ let mut attr = AttributeValue::default(); @@ -74,14 +75,15 @@ macro_rules! val { /// assert_eq!(map.get("c"), None); /// # } /// ``` +#[macro_export] macro_rules! hashmap { (@single $($x:tt)*) => (()); - (@count $($rest:expr),*) => (<[()]>::len(&[$(hashmap!(@single $rest)),*])); + (@count $($rest:expr),*) => (<[()]>::len(&[$($crate::hashmap!(@single $rest)),*])); - ($($key:expr => $value:expr,)+) => { hashmap!($($key => $value),+) }; + ($($key:expr => $value:expr,)+) => { $crate::hashmap!($($key => $value),+) }; ($($key:expr => $value:expr),*) => { { - let _cap = hashmap!(@count $($key),*); + let _cap = $crate::hashmap!(@count $($key),*); let mut _map = ::std::collections::HashMap::with_capacity(_cap); $( _map.insert($key, $value); @@ -92,11 +94,12 @@ macro_rules! hashmap { } /// Shorthand for specifying a dynamodb item +#[macro_export] macro_rules! ddb_item { ($($p:tt: $t:tt => $x:expr),*) => { { use rusoto_dynamodb::AttributeValue; - hashmap!{ + $crate::hashmap!{ $( String::from(stringify!($p)) => AttributeValue { $t: Some($x), From 580f245958c17681fcb5c7130167680aa1e16134 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Fri, 10 Jul 2020 13:35:06 -0400 Subject: [PATCH 2/5] Don't expose internal database error messages in responses --- autoendpoint/src/db/error.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autoendpoint/src/db/error.rs b/autoendpoint/src/db/error.rs index 05d43084c..8338d3f71 100644 --- a/autoendpoint/src/db/error.rs +++ b/autoendpoint/src/db/error.rs @@ -7,16 +7,16 @@ pub type DbResult = Result; #[derive(Debug, Error)] pub enum DbError { - #[error(transparent)] + #[error("Database error while performing GetItem")] GetItem(#[from] RusotoError), - #[error(transparent)] + #[error("Database error while performing UpdateItem")] UpdateItem(#[from] RusotoError), - #[error(transparent)] + #[error("Database error while performing PutItem")] PutItem(#[from] RusotoError), - #[error(transparent)] + #[error("Database error while performing DeleteItem")] DeleteItem(#[from] RusotoError), #[error("Error while deserializing database response: {0}")] From 7a19efd8553428899c6e663e83ce367bd9dfb194 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Fri, 10 Jul 2020 13:49:11 -0400 Subject: [PATCH 3/5] Fix hashmap macro example --- autopush-common/src/db/macros.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/autopush-common/src/db/macros.rs b/autopush-common/src/db/macros.rs index 2e514ad91..500b79c8c 100644 --- a/autopush-common/src/db/macros.rs +++ b/autopush-common/src/db/macros.rs @@ -63,8 +63,7 @@ macro_rules! val { /// ## Example /// /// ``` -/// #[macro_use] extern crate rusoto_helpers; -/// # fn main() { +/// use autopush_common::hashmap; /// /// let map = hashmap!{ /// "a" => 1, @@ -73,7 +72,6 @@ macro_rules! val { /// assert_eq!(map["a"], 1); /// assert_eq!(map["b"], 2); /// assert_eq!(map.get("c"), None); -/// # } /// ``` #[macro_export] macro_rules! hashmap { From 663442eeca93f89bdd1a9af703c963120edf1afa Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Fri, 10 Jul 2020 13:51:21 -0400 Subject: [PATCH 4/5] Enable the log feature of "again" --- Cargo.lock | 1 + autoendpoint/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 37667f06d..4a83c9bff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -266,6 +266,7 @@ name = "again" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "wasm-timer 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/autoendpoint/Cargo.toml b/autoendpoint/Cargo.toml index ae973dc35..2a4664000 100644 --- a/autoendpoint/Cargo.toml +++ b/autoendpoint/Cargo.toml @@ -9,7 +9,7 @@ actix-http = "1.0" actix-web = "2.0" actix-rt = "1.0" actix-cors = "0.2.0" -again = { version = "0.1.2", default-features = false } +again = { version = "0.1.2", default-features = false, features = ["log"] } async-trait = "0.1.36" autopush_common = { path = "../autopush-common" } backtrace = "0.3" From 89813f669ad5b5daf76119e3e499f46ea4a8dcbd Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Tue, 21 Jul 2020 09:12:15 -0400 Subject: [PATCH 5/5] Handle both serialization AND deserialization errors in DbClient --- autoendpoint/src/db/client.rs | 3 +-- autoendpoint/src/db/error.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/autoendpoint/src/db/client.rs b/autoendpoint/src/db/client.rs index f00b31f9f..4222d834d 100644 --- a/autoendpoint/src/db/client.rs +++ b/autoendpoint/src/db/client.rs @@ -166,8 +166,7 @@ impl DbClient { /// Store a single message pub async fn store_message(&self, uaid: Uuid, message: Notification) -> DbResult<()> { let put_item = PutItemInput { - item: serde_dynamodb::to_hashmap(&DynamoDbNotification::from_notif(&uaid, message)) - .unwrap(), + item: serde_dynamodb::to_hashmap(&DynamoDbNotification::from_notif(&uaid, message))?, table_name: self.message_table.clone(), ..Default::default() }; diff --git a/autoendpoint/src/db/error.rs b/autoendpoint/src/db/error.rs index 8338d3f71..ddbfc5898 100644 --- a/autoendpoint/src/db/error.rs +++ b/autoendpoint/src/db/error.rs @@ -19,6 +19,6 @@ pub enum DbError { #[error("Database error while performing DeleteItem")] DeleteItem(#[from] RusotoError), - #[error("Error while deserializing database response: {0}")] - Deserialize(#[from] serde_dynamodb::Error), + #[error("Error while performing (de)serialization: {0}")] + Serialization(#[from] serde_dynamodb::Error), }