diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 4127e3904..ec635098b 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -1,7 +1,4 @@ [advisories] ignore = [ - "RUSTSEC-2021-0124", # Bound by tokio restrictions, rusoto, reqwest, hyper... - "RUSTSEC-2023-0052", # Bound by Rusoto 0.47, Rustls 0.20, hyper-rustls 0.22, a2 0.8 - "RUSTSEC-2024-0336", # Bound by hyper-alpn 0.4.1, hyper-rustls 0.22.0/0.23.4, - # tokio-rustls 0.22.0/0.23.4 (not affected) + "RUSTSEC-2021-0124", # Bound by tokio restrictions, cargo, reqwest, hyper... ] diff --git a/.dockerignore b/.dockerignore index 0b95bd847..d44f5ead1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,7 +18,6 @@ html_coverage site-packages/* lib-python/* bin/* -ddb include/* lib_pypy/* pypy diff --git a/.gitignore b/.gitignore index 7a9d50527..f72d9d619 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ syntax:glob *.egg *~ build -ddb dist docs/_build *.xml diff --git a/Cargo.lock b/Cargo.lock index 96fd669f7..2199f4336 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,7 +22,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -35,7 +35,7 @@ dependencies = [ "actix-rt", "actix_derive", "bitflags 2.5.0", - "bytes 1.6.0", + "bytes", "crossbeam-channel", "futures-core", "futures-sink", @@ -44,9 +44,9 @@ dependencies = [ "log", "once_cell", "parking_lot 0.12.3", - "pin-project-lite 0.2.14", + "pin-project-lite", "smallvec", - "tokio 1.38.0", + "tokio", "tokio-util", ] @@ -57,12 +57,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ "bitflags 2.5.0", - "bytes 1.6.0", + "bytes", "futures-core", "futures-sink", "memchr", - "pin-project-lite 0.2.14", - "tokio 1.38.0", + "pin-project-lite", + "tokio", "tokio-util", "tracing", ] @@ -96,7 +96,7 @@ dependencies = [ "base64 0.22.1", "bitflags 2.5.0", "brotli", - "bytes 1.6.0", + "bytes", "bytestring", "derive_more", "encoding_rs", @@ -111,11 +111,11 @@ dependencies = [ "local-channel", "mime", "percent-encoding", - "pin-project-lite 0.2.14", + "pin-project-lite", "rand 0.8.5", "sha1", "smallvec", - "tokio 1.38.0", + "tokio", "tokio-util", "tracing", "zstd", @@ -134,7 +134,7 @@ dependencies = [ "actix-tls", "actix-utils", "awc", - "bytes 1.6.0", + "bytes", "futures-core", "http 0.2.12", "log", @@ -143,7 +143,7 @@ dependencies = [ "serde_urlencoded", "slab", "socket2", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -179,7 +179,7 @@ checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" dependencies = [ "actix-macros", "futures-core", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -195,7 +195,7 @@ dependencies = [ "futures-util", "mio", "socket2", - "tokio 1.38.0", + "tokio", "tracing", ] @@ -207,7 +207,7 @@ checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" dependencies = [ "futures-core", "paste", - "pin-project-lite 0.2.14", + "pin-project-lite", ] [[package]] @@ -230,7 +230,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -246,8 +246,8 @@ dependencies = [ "http 0.2.12", "http 1.1.0", "impl-more", - "pin-project-lite 0.2.14", - "tokio 1.38.0", + "pin-project-lite", + "tokio", "tokio-util", "tracing", ] @@ -259,7 +259,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" dependencies = [ "local-waker", - "pin-project-lite 0.2.14", + "pin-project-lite", ] [[package]] @@ -278,7 +278,7 @@ dependencies = [ "actix-utils", "actix-web-codegen", "ahash", - "bytes 1.6.0", + "bytes", "bytestring", "cfg-if", "cookie", @@ -291,7 +291,7 @@ dependencies = [ "log", "mime", "once_cell", - "pin-project-lite 0.2.14", + "pin-project-lite", "regex", "regex-lite", "serde", @@ -325,7 +325,7 @@ dependencies = [ "actix-http", "actix-web", "futures-core", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -505,7 +505,7 @@ checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", - "pin-project-lite 0.2.14", + "pin-project-lite", ] [[package]] @@ -617,7 +617,7 @@ dependencies = [ "serde_json", "slog", "slog-scope", - "tokio 1.38.0", + "tokio", "uuid", ] @@ -638,7 +638,7 @@ dependencies = [ "serde_json", "slog", "slog-scope", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -656,7 +656,7 @@ dependencies = [ "autoconnect_ws", "autopush_common", "backtrace", - "bytes 1.6.0", + "bytes", "bytestring", "cadence", "ctor", @@ -665,7 +665,7 @@ dependencies = [ "serde_json", "slog-scope", "thiserror", - "tokio 1.38.0", + "tokio", "uuid", ] @@ -692,7 +692,7 @@ dependencies = [ "slog-scope", "strum", "thiserror", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -715,7 +715,7 @@ dependencies = [ "serde_json", "slog-scope", "thiserror", - "tokio 1.38.0", + "tokio", "uuid", ] @@ -756,7 +756,6 @@ dependencies = [ "sentry-core", "serde", "serde_derive", - "serde_dynamodb", "serde_json", "slog", "slog-async", @@ -766,7 +765,7 @@ dependencies = [ "slog-term", "tempfile", "thiserror", - "tokio 1.38.0", + "tokio", "url", "uuid", "validator", @@ -807,13 +806,10 @@ dependencies = [ "rand 0.8.5", "regex", "reqwest 0.12.5", - "rusoto_core", - "rusoto_dynamodb", "sentry", "sentry-backtrace", "serde", "serde_derive", - "serde_dynamodb", "serde_json", "slog", "slog-async", @@ -824,7 +820,7 @@ dependencies = [ "slog-term", "tempfile", "thiserror", - "tokio 0.2.25", + "tokio", "url", "uuid", "woothee", @@ -843,7 +839,7 @@ dependencies = [ "actix-tls", "actix-utils", "base64 0.22.1", - "bytes 1.6.0", + "bytes", "cfg-if", "cookie", "derive_more", @@ -855,12 +851,12 @@ dependencies = [ "log", "mime", "percent-encoding", - "pin-project-lite 0.2.14", + "pin-project-lite", "rand 0.8.5", "serde", "serde_json", "serde_urlencoded", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -878,12 +874,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.7" @@ -934,15 +924,6 @@ dependencies = [ "serde", ] -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -1003,12 +984,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - [[package]] name = "bytes" version = "1.6.0" @@ -1021,7 +996,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" dependencies = [ - "bytes 1.6.0", + "bytes", ] [[package]] @@ -1069,7 +1044,6 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", - "serde", "wasm-bindgen", "windows-targets 0.52.5", ] @@ -1256,25 +1230,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-mac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" -dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "ct-logs" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" -dependencies = [ - "sct 0.6.1", -] - [[package]] name = "ctor" version = "0.2.8" @@ -1329,7 +1284,7 @@ dependencies = [ "async-trait", "deadpool-runtime", "num_cpus", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -1338,7 +1293,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" dependencies = [ - "tokio 1.38.0", + "tokio", ] [[package]] @@ -1374,22 +1329,13 @@ dependencies = [ "syn 2.0.68", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "crypto-common", ] @@ -1656,7 +1602,7 @@ checksum = "45ec6fe3675af967e67c5536c0b9d44e34e6c52f86bedc4ea49c5317b8e94d06" dependencies = [ "futures-channel", "futures-task", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -1696,7 +1642,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.14", + "pin-project-lite", "pin-utils", "slab", ] @@ -1805,7 +1751,7 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ - "bytes 1.6.0", + "bytes", "fnv", "futures-core", "futures-sink", @@ -1813,7 +1759,7 @@ dependencies = [ "http 0.2.12", "indexmap", "slab", - "tokio 1.38.0", + "tokio", "tokio-util", "tracing", ] @@ -1825,14 +1771,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ "atomic-waker", - "bytes 1.6.0", + "bytes", "fnv", "futures-core", "futures-sink", "http 1.1.0", "indexmap", "slab", - "tokio 1.38.0", + "tokio", "tokio-util", "tracing", ] @@ -1876,16 +1822,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hmac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" -dependencies = [ - "crypto-mac", - "digest 0.9.0", -] - [[package]] name = "home" version = "0.5.9" @@ -1923,7 +1859,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "bytes 1.6.0", + "bytes", "fnv", "itoa", ] @@ -1934,7 +1870,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ - "bytes 1.6.0", + "bytes", "fnv", "itoa", ] @@ -1945,9 +1881,9 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ - "bytes 1.6.0", + "bytes", "http 0.2.12", - "pin-project-lite 0.2.14", + "pin-project-lite", ] [[package]] @@ -1956,7 +1892,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ - "bytes 1.6.0", + "bytes", "http 1.1.0", ] @@ -1966,11 +1902,11 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ - "bytes 1.6.0", + "bytes", "futures-util", "http 1.1.0", "http-body 1.0.0", - "pin-project-lite 0.2.14", + "pin-project-lite", ] [[package]] @@ -1997,7 +1933,7 @@ version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ - "bytes 1.6.0", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -2007,9 +1943,9 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.14", + "pin-project-lite", "socket2", - "tokio 1.38.0", + "tokio", "tower-service", "tracing", "want", @@ -2021,7 +1957,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ - "bytes 1.6.0", + "bytes", "futures-channel", "futures-util", "h2 0.4.5", @@ -2029,29 +1965,12 @@ dependencies = [ "http-body 1.0.0", "httparse", "itoa", - "pin-project-lite 0.2.14", + "pin-project-lite", "smallvec", - "tokio 1.38.0", + "tokio", "want", ] -[[package]] -name = "hyper-rustls" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" -dependencies = [ - "ct-logs", - "futures-util", - "hyper 0.14.29", - "log", - "rustls 0.19.1", - "rustls-native-certs 0.5.0", - "tokio 1.38.0", - "tokio-rustls 0.22.0", - "webpki", -] - [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2063,8 +1982,8 @@ dependencies = [ "hyper 0.14.29", "log", "rustls 0.21.12", - "rustls-native-certs 0.6.3", - "tokio 1.38.0", + "rustls-native-certs", + "tokio", "tokio-rustls 0.24.1", ] @@ -2080,7 +1999,7 @@ dependencies = [ "hyper-util", "rustls 0.22.4", "rustls-pki-types", - "tokio 1.38.0", + "tokio", "tokio-rustls 0.25.0", "tower-service", "webpki-roots", @@ -2098,7 +2017,7 @@ dependencies = [ "hyper-util", "rustls 0.23.10", "rustls-pki-types", - "tokio 1.38.0", + "tokio", "tokio-rustls 0.26.0", "tower-service", ] @@ -2109,10 +2028,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.6.0", + "bytes", "hyper 0.14.29", "native-tls", - "tokio 1.38.0", + "tokio", "tokio-native-tls", ] @@ -2122,12 +2041,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ - "bytes 1.6.0", + "bytes", "http-body-util", "hyper 1.3.1", "hyper-util", "native-tls", - "tokio 1.38.0", + "tokio", "tokio-native-tls", "tower-service", ] @@ -2138,15 +2057,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ - "bytes 1.6.0", + "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.0", "hyper 1.3.1", - "pin-project-lite 0.2.14", + "pin-project-lite", "socket2", - "tokio 1.38.0", + "tokio", "tower", "tower-service", "tracing", @@ -2292,7 +2211,7 @@ dependencies = [ "base64 0.21.7", "js-sys", "pem", - "ring 0.17.8", + "ring", "serde", "serde_json", "simple_asn1", @@ -2405,17 +2324,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" -[[package]] -name = "md-5" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" -dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "memchr" version = "2.7.4" @@ -2516,7 +2424,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "similar", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -2625,12 +2533,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "openssl" version = "0.10.64" @@ -2820,7 +2722,7 @@ checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" dependencies = [ "once_cell", "pest", - "sha2 0.10.8", + "sha2", ] [[package]] @@ -2843,12 +2745,6 @@ dependencies = [ "syn 2.0.68", ] -[[package]] -name = "pin-project-lite" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" - [[package]] name = "pin-project-lite" version = "0.2.14" @@ -3095,7 +2991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "base64 0.21.7", - "bytes 1.6.0", + "bytes", "encoding_rs", "futures-core", "futures-util", @@ -3111,14 +3007,14 @@ dependencies = [ "native-tls", "once_cell", "percent-encoding", - "pin-project-lite 0.2.14", + "pin-project-lite", "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", "system-configuration", - "tokio 1.38.0", + "tokio", "tokio-native-tls", "tower-service", "url", @@ -3135,7 +3031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ "base64 0.22.1", - "bytes 1.6.0", + "bytes", "encoding_rs", "futures-channel", "futures-core", @@ -3155,14 +3051,14 @@ dependencies = [ "native-tls", "once_cell", "percent-encoding", - "pin-project-lite 0.2.14", + "pin-project-lite", "rustls-pemfile 2.1.2", "serde", "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", "system-configuration", - "tokio 1.38.0", + "tokio", "tokio-native-tls", "tower-service", "url", @@ -3172,21 +3068,6 @@ dependencies = [ "winreg 0.52.0", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - [[package]] name = "ring" version = "0.17.8" @@ -3197,8 +3078,8 @@ dependencies = [ "cfg-if", "getrandom 0.2.15", "libc", - "spin 0.9.8", - "untrusted 0.9.0", + "spin", + "untrusted", "windows-sys 0.52.0", ] @@ -3214,89 +3095,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "rusoto_core" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b4f000e8934c1b4f70adde180056812e7ea6b1a247952db8ee98c94cd3116cc" -dependencies = [ - "async-trait", - "base64 0.13.1", - "bytes 1.6.0", - "crc32fast", - "futures 0.3.30", - "http 0.2.12", - "hyper 0.14.29", - "hyper-rustls 0.22.1", - "lazy_static", - "log", - "rusoto_credential", - "rusoto_signature", - "rustc_version", - "serde", - "serde_json", - "tokio 1.38.0", - "xml-rs", -] - -[[package]] -name = "rusoto_credential" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a46b67db7bb66f5541e44db22b0a02fed59c9603e146db3a9e633272d3bac2f" -dependencies = [ - "async-trait", - "chrono", - "dirs-next", - "futures 0.3.30", - "hyper 0.14.29", - "serde", - "serde_json", - "shlex", - "tokio 1.38.0", - "zeroize", -] - -[[package]] -name = "rusoto_dynamodb" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7935e1f9ca57c4ee92a4d823dcd698eb8c992f7e84ca21976ae72cd2b03016e7" -dependencies = [ - "async-trait", - "bytes 1.6.0", - "futures 0.3.30", - "rusoto_core", - "serde", - "serde_json", -] - -[[package]] -name = "rusoto_signature" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6264e93384b90a747758bcc82079711eacf2e755c3a8b5091687b5349d870bcc" -dependencies = [ - "base64 0.13.1", - "bytes 1.6.0", - "chrono", - "digest 0.9.0", - "futures 0.3.30", - "hex", - "hmac", - "http 0.2.12", - "hyper 0.14.29", - "log", - "md-5", - "percent-encoding", - "pin-project-lite 0.2.14", - "rusoto_credential", - "rustc_version", - "serde", - "sha2 0.9.9", - "tokio 1.38.0", -] - [[package]] name = "rust-ini" version = "0.19.0" @@ -3341,19 +3139,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustls" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" -dependencies = [ - "base64 0.13.1", - "log", - "ring 0.16.20", - "sct 0.6.1", - "webpki", -] - [[package]] name = "rustls" version = "0.21.12" @@ -3361,9 +3146,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.17.8", + "ring", "rustls-webpki 0.101.7", - "sct 0.7.1", + "sct", ] [[package]] @@ -3373,7 +3158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ "log", - "ring 0.17.8", + "ring", "rustls-pki-types", "rustls-webpki 0.102.4", "subtle", @@ -3393,18 +3178,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" -dependencies = [ - "openssl-probe", - "rustls 0.19.1", - "schannel", - "security-framework", -] - [[package]] name = "rustls-native-certs" version = "0.6.3" @@ -3448,8 +3221,8 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -3458,9 +3231,9 @@ version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ - "ring 0.17.8", + "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -3499,24 +3272,14 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", -] - [[package]] name = "sct" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -3570,7 +3333,7 @@ dependencies = [ "sentry-debug-images", "sentry-panic", "sentry-tracing", - "tokio 1.38.0", + "tokio", "ureq", ] @@ -3695,17 +3458,6 @@ dependencies = [ "syn 2.0.68", ] -[[package]] -name = "serde_dynamodb" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0a485bf5e371e28d4978444686ffa91830d4719b92fb31960fbd8f494816239" -dependencies = [ - "bytes 1.6.0", - "rusoto_dynamodb", - "serde", -] - [[package]] name = "serde_json" version = "1.0.117" @@ -3746,20 +3498,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", -] - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", + "digest", ] [[package]] @@ -3770,7 +3509,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -3914,12 +3653,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -3979,7 +3712,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", - "quote", "unicode-ident", ] @@ -4167,17 +3899,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "tokio" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" -dependencies = [ - "bytes 0.5.6", - "pin-project-lite 0.1.12", - "tokio-macros 0.2.6", -] - [[package]] name = "tokio" version = "1.38.0" @@ -4185,29 +3906,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", - "bytes 1.6.0", + "bytes", "libc", "mio", - "num_cpus", "parking_lot 0.12.3", - "pin-project-lite 0.2.14", + "pin-project-lite", "signal-hook-registry", "socket2", - "tokio-macros 2.3.0", + "tokio-macros", "windows-sys 0.48.0", ] -[[package]] -name = "tokio-macros" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "tokio-macros" version = "2.3.0" @@ -4226,18 +3935,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.38.0", -] - -[[package]] -name = "tokio-rustls" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" -dependencies = [ - "rustls 0.19.1", - "tokio 1.38.0", - "webpki", + "tokio", ] [[package]] @@ -4247,7 +3945,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls 0.21.12", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -4258,7 +3956,7 @@ checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ "rustls 0.22.4", "rustls-pki-types", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -4269,7 +3967,7 @@ checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ "rustls 0.23.10", "rustls-pki-types", - "tokio 1.38.0", + "tokio", ] [[package]] @@ -4278,11 +3976,11 @@ version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ - "bytes 1.6.0", + "bytes", "futures-core", "futures-sink", - "pin-project-lite 0.2.14", - "tokio 1.38.0", + "pin-project-lite", + "tokio", ] [[package]] @@ -4328,8 +4026,8 @@ dependencies = [ "futures-core", "futures-util", "pin-project", - "pin-project-lite 0.2.14", - "tokio 1.38.0", + "pin-project-lite", + "tokio", "tower-layer", "tower-service", ] @@ -4353,7 +4051,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "log", - "pin-project-lite 0.2.14", + "pin-project-lite", "tracing-core", ] @@ -4436,12 +4134,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -4664,16 +4356,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", -] - [[package]] name = "webpki-roots" version = "0.26.3" @@ -4923,12 +4605,6 @@ dependencies = [ "regex", ] -[[package]] -name = "xml-rs" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" - [[package]] name = "yaml-rust" version = "0.4.5" @@ -4960,7 +4636,7 @@ dependencies = [ "serde", "serde_json", "time", - "tokio 1.38.0", + "tokio", "tower-service", "url", ] diff --git a/Cargo.toml b/Cargo.toml index 23a3bb167..ce182b2c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,12 +68,6 @@ openssl = { version = "0.10" } rand = "0.8" regex = "1.4" reqwest = { version = "0.12", features = ["json", "blocking"] } -rusoto_core = { version = "0.47", default-features = false, features = [ - "rustls", -] } # locked by serde_dynamodb 0.9 -rusoto_dynamodb = { version = "0.47", default-features = false, features = [ - "rustls", -] } # locked by serde_dynamodb 0.9 sentry = { version = "0.32", features = [ "debug-logs", ] } # Using debug-logs avoids https://github.com/getsentry/sentry-rust/issues/237 @@ -82,7 +76,6 @@ sentry-actix = "0.32" sentry-backtrace = "0.32" serde = "1.0" serde_derive = "1.0" -serde_dynamodb = "0.9" serde_json = "1.0" slog = { version = "2.7", features = [ "dynamic-keys", @@ -96,7 +89,7 @@ slog-scope = "4.4" slog-stdlog = "4.1" slog-term = "2.6" thiserror = "1.0" -tokio = "1.36" +tokio = "1.38" tokio-compat-02 = "0.2" tokio-core = "0.1" tokio-io = "0.1" diff --git a/autoconnect/Cargo.toml b/autoconnect/Cargo.toml index a53925866..c0ba4ba27 100644 --- a/autoconnect/Cargo.toml +++ b/autoconnect/Cargo.toml @@ -53,8 +53,7 @@ actix-service = "2.0" docopt = "1.1" [features] -default = ["bigtable", "emulator"] +default = ["bigtable"] bigtable = ["autopush_common/bigtable", "autoconnect_settings/bigtable"] -dynamodb = ["autopush_common/dynamodb", "autoconnect_settings/dynamodb"] emulator = ["bigtable"] log_vapid = [] diff --git a/autoconnect/autoconnect-settings/Cargo.toml b/autoconnect/autoconnect-settings/Cargo.toml index dc99ab284..32c04aeb0 100644 --- a/autoconnect/autoconnect-settings/Cargo.toml +++ b/autoconnect/autoconnect-settings/Cargo.toml @@ -23,6 +23,5 @@ autopush_common.workspace = true [features] # specify the default via the calling crate, in order to simplify default chains. -dynamodb = [] bigtable = ["autopush_common/bigtable"] emulator = ["bigtable"] diff --git a/autoconnect/autoconnect-settings/src/app_state.rs b/autoconnect/autoconnect-settings/src/app_state.rs index fd82287ef..0d7656eca 100644 --- a/autoconnect/autoconnect-settings/src/app_state.rs +++ b/autoconnect/autoconnect-settings/src/app_state.rs @@ -2,8 +2,6 @@ use std::{sync::Arc, time::Duration}; #[cfg(feature = "bigtable")] use autopush_common::db::bigtable::BigTableClientImpl; -#[cfg(feature = "dynamodb")] -use autopush_common::db::dynamodb::DdbClientImpl; use cadence::StatsdClient; use config::ConfigError; use fernet::{Fernet, MultiFernet}; @@ -72,11 +70,6 @@ impl AppState { let storage_type = StorageType::from_dsn(&db_settings.dsn); #[allow(unused)] let db: Box = match storage_type { - #[cfg(feature = "dynamodb")] - StorageType::DynamoDb => Box::new( - DdbClientImpl::new(metrics.clone(), &db_settings) - .map_err(|e| ConfigError::Message(e.to_string()))?, - ), #[cfg(feature = "bigtable")] StorageType::BigTable => { let client = BigTableClientImpl::new(metrics.clone(), &db_settings) diff --git a/autoconnect/autoconnect-ws/Cargo.toml b/autoconnect/autoconnect-ws/Cargo.toml index 9bfdf773c..b6b08accf 100644 --- a/autoconnect/autoconnect-ws/Cargo.toml +++ b/autoconnect/autoconnect-ws/Cargo.toml @@ -18,7 +18,7 @@ serde_json.workspace = true sentry.workspace = true slog-scope.workspace = true thiserror.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["macros"] } async-trait = "0.1" strum = { version = "0.26", features = ["derive"] } diff --git a/autoconnect/autoconnect-ws/autoconnect-ws-sm/src/identified/on_server_notif.rs b/autoconnect/autoconnect-ws/autoconnect-ws-sm/src/identified/on_server_notif.rs index 6ee206427..7e6b6f62f 100644 --- a/autoconnect/autoconnect-ws/autoconnect-ws-sm/src/identified/on_server_notif.rs +++ b/autoconnect/autoconnect-ws/autoconnect-ws-sm/src/identified/on_server_notif.rs @@ -181,9 +181,8 @@ impl WebPushClient { .or(self.current_timestamp); trace!("🗄️ WebPushClient::do_check_storage {:?}", ×tamp); // if we're to include topic messages, do those first. - // NOTE: DynamoDB would also wind up fetching the `current_timestamp` when pulling these, - // Bigtable can't fetch `current_timestamp` so we can't rely on `fetch_topic_messages()` - // returning a reasonable timestamp. + // NOTE: Bigtable can't fetch `current_timestamp`, so we can't rely on + // `fetch_topic_messages()` returning a reasonable timestamp. let topic_resp = if self.flags.include_topic { trace!("🗄️ WebPushClient::do_check_storage: fetch_topic_messages"); // Get the most recent max 11 messages. diff --git a/autoconnect/autoconnect-ws/autoconnect-ws-sm/src/unidentified.rs b/autoconnect/autoconnect-ws/autoconnect-ws-sm/src/unidentified.rs index 8a6212f06..9fcc71a24 100644 --- a/autoconnect/autoconnect-ws/autoconnect-ws-sm/src/unidentified.rs +++ b/autoconnect/autoconnect-ws/autoconnect-ws-sm/src/unidentified.rs @@ -124,9 +124,6 @@ impl UnidentifiedClient { let connected_at = ms_since_epoch(); if let Some(uaid) = uaid { - // NOTE: previously a user would be dropped when - // serde_dynamodb::from_hashmap failed (but this now occurs inside - // the db impl) if let Some(mut user) = self.app_state.db.get_user(&uaid).await? { let flags = ClientFlags { check_storage: true, diff --git a/autoendpoint/Cargo.toml b/autoendpoint/Cargo.toml index c0e7f5ac6..9e10be4b9 100644 --- a/autoendpoint/Cargo.toml +++ b/autoendpoint/Cargo.toml @@ -29,7 +29,6 @@ sentry-actix.workspace = true sentry-core.workspace = true serde.workspace = true serde_derive.workspace = true -serde_dynamodb.workspace = true serde_json.workspace = true slog.workspace = true slog-async.workspace = true @@ -71,5 +70,4 @@ tokio = { workspace = true, features = ["fs", "macros"] } [features] default = ["bigtable"] bigtable = ["autopush_common/bigtable"] -dynamodb = ["autopush_common/dynamodb"] emulator = ["bigtable"] diff --git a/autoendpoint/src/extractors/routers.rs b/autoendpoint/src/extractors/routers.rs index 41facb205..f25e3792b 100644 --- a/autoendpoint/src/extractors/routers.rs +++ b/autoendpoint/src/extractors/routers.rs @@ -12,7 +12,6 @@ use std::fmt::{self, Display}; use std::str::FromStr; use std::sync::Arc; -/// Valid `DynamoDbUser::router_type` values #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[allow(clippy::upper_case_acronyms)] pub enum RouterType { diff --git a/autoendpoint/src/routers/apns/router.rs b/autoendpoint/src/routers/apns/router.rs index 2c7ca4450..745f3a897 100644 --- a/autoendpoint/src/routers/apns/router.rs +++ b/autoendpoint/src/routers/apns/router.rs @@ -234,21 +234,6 @@ impl ApnsRouter { ApiError::from(ApnsError::ApnsUpstream(error)) } - /// Convert all of the floats in a JSON value into integers. DynamoDB - /// returns all numbers as floats, but deserializing to `APS` will fail if - /// it expects an integer and gets a float. - fn convert_value_float_to_int(value: &mut Value) { - if let Some(float) = value.as_f64() { - *value = Value::Number(serde_json::Number::from(float as i64)); - } - - if let Some(object) = value.as_object_mut() { - object - .values_mut() - .for_each(Self::convert_value_float_to_int); - } - } - /// if we have any clients defined, this connection is "active" pub fn active(&self) -> bool { !self.clients.is_empty() @@ -432,13 +417,7 @@ impl Router for ApnsRouter { .get("rel_channel") .and_then(Value::as_str) .ok_or(ApnsError::NoReleaseChannel)?; - // XXX: We don't really use anything that is a numeric here, aside from - // mutable contant, and even there we should just check for presense. - // Once we're off of DynamoDB, we might want to kill the map. - let aps_json = router_data.get("aps").cloned().map(|mut value| { - Self::convert_value_float_to_int(&mut value); - value - }); + let aps_json = router_data.get("aps").cloned(); let mut message_data = build_message_data(notification)?; message_data.insert("ver", notification.message_id.clone()); diff --git a/autoendpoint/src/server.rs b/autoendpoint/src/server.rs index 5edfaefda..4010b233d 100644 --- a/autoendpoint/src/server.rs +++ b/autoendpoint/src/server.rs @@ -13,8 +13,6 @@ use serde_json::json; #[cfg(feature = "bigtable")] use autopush_common::db::bigtable::BigTableClientImpl; -#[cfg(feature = "dynamodb")] -use autopush_common::db::dynamodb::DdbClientImpl; use autopush_common::{ db::{client::DbClient, spawn_pool_periodic_reporter, DbSettings, StorageType}, middleware::sentry::SentryWrapper, @@ -64,11 +62,6 @@ impl Server { }, }; let db: Box = match StorageType::from_dsn(&db_settings.dsn) { - #[cfg(feature = "dynamodb")] - StorageType::DynamoDb => { - debug!("Using Dynamodb"); - Box::new(DdbClientImpl::new(metrics.clone(), &db_settings)?) - } #[cfg(feature = "bigtable")] StorageType::BigTable => { debug!("Using BigTable"); diff --git a/autopush-common/Cargo.toml b/autopush-common/Cargo.toml index f8a763646..8a0770cff 100644 --- a/autopush-common/Cargo.toml +++ b/autopush-common/Cargo.toml @@ -27,13 +27,10 @@ openssl.workspace = true rand.workspace = true regex.workspace = true reqwest.workspace = true -rusoto_core = { workspace = true, optional = true } -rusoto_dynamodb = { workspace = true, optional = true } sentry-backtrace.workspace = true sentry.workspace = true serde.workspace = true serde_derive.workspace = true -serde_dynamodb.workspace = true serde_json.workspace = true slog.workspace = true slog-async.workspace = true @@ -65,7 +62,7 @@ form_urlencoded = { version = "1.2", optional = true } [dev-dependencies] mockito = "0.31" tempfile = "3.2.0" -tokio = { version = "0.2", features = ["macros"] } +tokio = { workspace=true, features = ["macros"] } actix-rt = "2.8" [features] @@ -78,7 +75,6 @@ bigtable = [ "dep:protobuf", "dep:form_urlencoded", ] -dynamodb = ["dep:rusoto_core", "dep:rusoto_dynamodb"] emulator = [ "bigtable", ] # used for testing big table, requires an external bigtable emulator running. diff --git a/autopush-common/build.rs b/autopush-common/build.rs index d2a3f84c1..06440274b 100644 --- a/autopush-common/build.rs +++ b/autopush-common/build.rs @@ -1,5 +1,5 @@ pub fn main() { - if !(cfg!(feature = "dynamodb") || cfg!(feature = "bigtable")) { - panic!("No database defined! Please compile with either `features=dynamodb` or `features=bigtable`"); + if !cfg!(feature = "bigtable") { + panic!("No database defined! Please compile with `features=bigtable`"); } } diff --git a/autopush-common/src/db/bigtable/bigtable_client/mod.rs b/autopush-common/src/db/bigtable/bigtable_client/mod.rs index f82f38ff5..621187864 100644 --- a/autopush-common/src/db/bigtable/bigtable_client/mod.rs +++ b/autopush-common/src/db/bigtable/bigtable_client/mod.rs @@ -1249,10 +1249,9 @@ impl DbClient for BigTableClientImpl { ); let messages = self.rows_to_notifications(rows)?; - // Note: Bigtable always returns a timestamp of None here whereas - // DynamoDB returns the `current_timestamp` read from its meta - // record. Under Bigtable `current_timestamp` is instead initially read - // from [get_user] + // Note: Bigtable always returns a timestamp of None. + // Under Bigtable `current_timestamp` is instead initially read + // from [get_user]. Ok(FetchMessageResponse { messages, timestamp: None, diff --git a/autopush-common/src/db/bigtable/mod.rs b/autopush-common/src/db/bigtable/mod.rs index d5e64867b..3d091013a 100644 --- a/autopush-common/src/db/bigtable/mod.rs +++ b/autopush-common/src/db/bigtable/mod.rs @@ -150,7 +150,7 @@ impl TryFrom<&str> for BigTableDbSettings { type Error = DbError; fn try_from(setting_string: &str) -> Result { let me: Self = serde_json::from_str(setting_string) - .map_err(|e| DbError::General(format!("Could not parse DdbSettings: {:?}", e)))?; + .map_err(|e| DbError::General(format!("Could not parse DbSettings: {:?}", e)))?; if me.table_name.starts_with('/') { return Err(DbError::ConnectionError( diff --git a/autopush-common/src/db/dynamodb/macros.rs b/autopush-common/src/db/dynamodb/macros.rs deleted file mode 100644 index bb944e0f2..000000000 --- a/autopush-common/src/db/dynamodb/macros.rs +++ /dev/null @@ -1,113 +0,0 @@ -/// A bunch of macro helpers from rusoto_helpers code, which they pulled from crates.io because -/// they were waiting for rusuto to hit 1.0.0 or something. For sanity, they are instead accumulated -/// here for our use. -#[allow(unused_macros)] -macro_rules! attributes { - ($($val:expr => $attr_type:expr),*) => { - { - let mut temp_vec = Vec::new(); - $( - temp_vec.push(AttributeDefinition { - attribute_name: String::from($val), - attribute_type: String::from($attr_type) - }); - )* - temp_vec - } - } -} - -#[allow(unused_macros)] -macro_rules! key_schema { - ($($name:expr => $key_type:expr),*) => { - { - let mut temp_vec = Vec::new(); - $( - temp_vec.push(KeySchemaElement { - key_type: String::from($key_type), - attribute_name: String::from($name) - }); - )* - temp_vec - } - } -} - -#[macro_export] -macro_rules! val { - (B => $val:expr) => { - AttributeValue { - b: Some($val), - ..Default::default() - } - }; - (S => $val:expr) => { - AttributeValue { - s: Some($val.to_string()), - ..Default::default() - } - }; - (SS => $val:expr) => { - AttributeValue { - ss: Some($val.iter().map(|v| v.to_string()).collect()), - ..Default::default() - } - }; - (N => $val:expr) => { - AttributeValue { - n: Some($val.to_string()), - ..Default::default() - } - }; -} - -/// Create a **HashMap** from a list of key-value pairs -/// -/// ## Example -/// -/// ```ignore -/// use autopush_common::hashmap; -/// -/// let map = hashmap!{ -/// "a" => 1, -/// "b" => 2, -/// }; -/// assert_eq!(map["a"], 1); -/// assert_eq!(map["b"], 2); -/// assert_eq!(map.get("c"), None); -/// ``` -#[macro_export] -macro_rules! hashmap { - (@single $($x:tt)*) => (()); - (@count $($rest:expr),*) => (<[()]>::len(&[$($crate::hashmap!(@single $rest)),*])); - - ($($key:expr => $value:expr,)+) => { $crate::hashmap!($($key => $value),+) }; - ($($key:expr => $value:expr),*) => { - { - let _cap = $crate::hashmap!(@count $($key),*); - let mut _map = ::std::collections::HashMap::with_capacity(_cap); - $( - _map.insert($key, $value); - )* - _map - } - }; -} - -/// Shorthand for specifying a dynamodb item -#[macro_export] -macro_rules! ddb_item { - ($($p:tt: $t:tt => $x:expr),*) => { - { - use rusoto_dynamodb::AttributeValue; - $crate::hashmap!{ - $( - String::from(stringify!($p)) => AttributeValue { - $t: Some($x), - ..Default::default() - }, - )* - } - } - } -} diff --git a/autopush-common/src/db/dynamodb/mod.rs b/autopush-common/src/db/dynamodb/mod.rs deleted file mode 100644 index e54846bd0..000000000 --- a/autopush-common/src/db/dynamodb/mod.rs +++ /dev/null @@ -1,648 +0,0 @@ -use std::collections::HashSet; -use std::env; -use std::fmt::{Debug, Display}; -use std::result::Result as StdResult; -use std::sync::Arc; - -use crate::db::client::DbClient; -use crate::db::dynamodb::retry::{ - retry_policy, retryable_batchwriteitem_error, retryable_delete_error, - retryable_describe_table_error, retryable_getitem_error, retryable_putitem_error, - retryable_query_error, retryable_updateitem_error, -}; -use crate::db::error::{DbError, DbResult}; -use crate::db::{ - client::FetchMessageResponse, DbSettings, NotificationRecord, User, MAX_CHANNEL_TTL, MAX_EXPIRY, -}; -use crate::notification::Notification; -use crate::util::sec_since_epoch; - -use async_trait::async_trait; -use cadence::{CountedExt, StatsdClient}; -use rusoto_core::credential::StaticProvider; -use rusoto_core::{HttpClient, Region, RusotoError}; -use rusoto_dynamodb::{ - AttributeValue, BatchWriteItemInput, DeleteItemInput, DescribeTableError, DescribeTableInput, - DynamoDb, DynamoDbClient, GetItemInput, ListTablesInput, PutItemInput, PutRequest, QueryInput, - UpdateItemError, UpdateItemInput, WriteRequest, -}; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -#[macro_use] -pub mod macros; -pub mod retry; - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct DynamoDbSettings { - #[serde(default)] - pub router_table: String, - #[serde(default)] - pub message_table: String, - #[serde(default)] - pub db_routing_table: Option, -} - -impl TryFrom<&str> for DynamoDbSettings { - type Error = DbError; - fn try_from(setting_string: &str) -> Result { - serde_json::from_str(setting_string) - .map_err(|e| DbError::General(format!("Could not parse DdbSettings: {:?}", e))) - } -} - -#[derive(Clone)] -pub struct DdbClientImpl { - db_client: DynamoDbClient, - metrics: Arc, - settings: DynamoDbSettings, -} - -impl DdbClientImpl { - pub fn new(metrics: Arc, db_settings: &DbSettings) -> DbResult { - debug!("🛢️DynamoDB Settings {:?}", db_settings); - let db_client = if let Ok(endpoint) = env::var("AWS_LOCAL_DYNAMODB") { - DynamoDbClient::new_with( - HttpClient::new().expect("TLS initialization error"), - StaticProvider::new_minimal("BogusKey".to_string(), "BogusKey".to_string()), - Region::Custom { - name: "us-east-1".to_string(), - endpoint, - }, - ) - } else { - DynamoDbClient::new(Region::default()) - }; - - let settings = DynamoDbSettings::try_from(db_settings.db_settings.as_ref())?; - Ok(Self { - db_client, - metrics, - settings, - }) - } - - /// Check if a table exists - async fn table_exists(&self, table_name: String) -> DbResult { - let input = DescribeTableInput { table_name }; - - let output = match retry_policy() - .retry_if( - || self.db_client.describe_table(input.clone()), - retryable_describe_table_error(self.metrics.clone()), - ) - .await - { - Ok(output) => output, - Err(RusotoError::Service(DescribeTableError::ResourceNotFound(_))) => { - return Ok(false); - } - Err(e) => return Err(e.into()), - }; - - let status = output - .table - .and_then(|table| table.table_status) - .ok_or(DbError::TableStatusUnknown)? - .to_uppercase(); - - Ok(["CREATING", "UPDATING", "ACTIVE"].contains(&status.as_str())) - } -} - -/// Like Result::ok, convert from Result to Option but applying a -/// function to the Err value -fn ok_or_inspect(result: StdResult, op: F) -> Option -where - F: FnOnce(E), -{ - match result { - Ok(t) => Some(t), - Err(e) => { - op(e); - None - } - } -} - -/// Log/metric errors during conversions to Notification -fn conversion_err(metrics: &StatsdClient, err: E, item: F, name: &'static str) -where - E: Display, - F: Debug, -{ - error!("Failed {}, item: {:?}, conversion: {}", name, item, err); - metrics - .incr_with_tags("ua.notification_read.error") - .with_tag("conversion", name) - .send(); -} - -#[allow(clippy::field_reassign_with_default)] -#[async_trait] -impl DbClient for DdbClientImpl { - async fn add_user(&self, user: &User) -> DbResult<()> { - let input = PutItemInput { - table_name: self.settings.router_table.clone(), - item: serde_dynamodb::to_hashmap(user)?, - condition_expression: Some("attribute_not_exists(uaid)".to_string()), - ..Default::default() - }; - - retry_policy() - .retry_if( - || self.db_client.put_item(input.clone()), - retryable_putitem_error(self.metrics.clone()), - ) - .await?; - Ok(()) - } - - async fn update_user(&self, user: &mut User) -> DbResult { - let mut user_map = serde_dynamodb::to_hashmap(&user)?; - user_map.remove("uaid"); - let input = UpdateItemInput { - table_name: self.settings.router_table.clone(), - key: ddb_item! { uaid: s => user.uaid.simple().to_string() }, - update_expression: Some(format!( - "SET {}", - user_map - .keys() - .map(|key| format!("{0}=:{0}", key)) - .collect::>() - .join(", ") - )), - expression_attribute_values: Some( - user_map - .into_iter() - .map(|(key, value)| (format!(":{}", key), value)) - .collect(), - ), - condition_expression: Some( - "attribute_exists(uaid) and ( - attribute_not_exists(router_type) or - (router_type = :router_type) - ) and ( - attribute_not_exists(node_id) or - (connected_at < :connected_at) - )" - .to_string(), - ), - ..Default::default() - }; - - let result = retry_policy() - .retry_if( - || self.db_client.update_item(input.clone()), - retryable_updateitem_error(self.metrics.clone()), - ) - .await; - match result { - Ok(_) => Ok(true), - Err(RusotoError::Service(UpdateItemError::ConditionalCheckFailed(_))) => Ok(false), - Err(e) => Err(e.into()), - } - } - - async fn get_user(&self, uaid: &Uuid) -> DbResult> { - let input = GetItemInput { - table_name: self.settings.router_table.clone(), - consistent_read: Some(true), - key: ddb_item! { uaid: s => uaid.simple().to_string() }, - ..Default::default() - }; - - retry_policy() - .retry_if( - || self.db_client.get_item(input.clone()), - retryable_getitem_error(self.metrics.clone()), - ) - .await? - .item - .map(serde_dynamodb::from_hashmap) - .transpose() - .map_err(|e| { - error!("DbClient::get_user: {:?}", e.to_string()); - DbError::from(e) - }) - } - - async fn remove_user(&self, uaid: &Uuid) -> DbResult<()> { - let input = DeleteItemInput { - table_name: self.settings.router_table.clone(), - key: ddb_item! { uaid: s => uaid.simple().to_string() }, - ..Default::default() - }; - - retry_policy() - .retry_if( - || self.db_client.delete_item(input.clone()), - retryable_delete_error(self.metrics.clone()), - ) - .await?; - Ok(()) - } - - async fn add_channel(&self, uaid: &Uuid, channel_id: &Uuid) -> DbResult<()> { - let input = UpdateItemInput { - table_name: self.settings.message_table.clone(), - key: ddb_item! { - uaid: s => uaid.simple().to_string(), - chidmessageid: s => " ".to_string() - }, - update_expression: Some("ADD chids :channel_id SET expiry = :expiry".to_string()), - expression_attribute_values: Some(hashmap! { - ":channel_id".to_string() => val!(SS => Some(channel_id)), - ":expiry".to_string() => val!(N => sec_since_epoch() + MAX_CHANNEL_TTL) - }), - ..Default::default() - }; - - retry_policy() - .retry_if( - || self.db_client.update_item(input.clone()), - retryable_updateitem_error(self.metrics.clone()), - ) - .await?; - Ok(()) - } - - /// Hopefully, this is never called. It is provided for completion sake. - async fn add_channels(&self, uaid: &Uuid, channels: HashSet) -> DbResult<()> { - for channel_id in channels { - self.add_channel(uaid, &channel_id).await?; - } - Ok(()) - } - - async fn get_channels(&self, uaid: &Uuid) -> DbResult> { - // Channel IDs are stored in a special row in the message table, where - // chidmessageid = " " - let input = GetItemInput { - table_name: self.settings.message_table.clone(), - consistent_read: Some(true), - key: ddb_item! { - uaid: s => uaid.simple().to_string(), - chidmessageid: s => " ".to_string() - }, - ..Default::default() - }; - - let output = retry_policy() - .retry_if( - || self.db_client.get_item(input.clone()), - retryable_getitem_error(self.metrics.clone()), - ) - .await?; - - // The channel IDs are in the notification's `chids` field - let channels = output - .item - // Deserialize the notification - .map(serde_dynamodb::from_hashmap::) - .transpose()? - // Extract the channel IDs - .and_then(|n| n.chids) - .unwrap_or_default(); - - // Convert the IDs from String to Uuid - let channels = channels - .into_iter() - .filter_map(|s| Uuid::parse_str(&s).ok()) - .collect(); - - Ok(channels) - } - - async fn remove_channel(&self, uaid: &Uuid, channel_id: &Uuid) -> DbResult { - let input = UpdateItemInput { - table_name: self.settings.message_table.clone(), - key: ddb_item! { - uaid: s => uaid.simple().to_string(), - chidmessageid: s => " ".to_string() - }, - update_expression: Some("DELETE chids :channel_id SET expiry = :expiry".to_string()), - expression_attribute_values: Some(hashmap! { - ":channel_id".to_string() => val!(SS => Some(channel_id)), - ":expiry".to_string() => val!(N => sec_since_epoch() + MAX_CHANNEL_TTL) - }), - return_values: Some("UPDATED_OLD".to_string()), - ..Default::default() - }; - - let output = retry_policy() - .retry_if( - || self.db_client.update_item(input.clone()), - retryable_updateitem_error(self.metrics.clone()), - ) - .await?; - - // Check if the old channel IDs contain the removed channel - Ok(output - .attributes - .as_ref() - .and_then(|map| map.get("chids")) - .and_then(|item| item.ss.as_ref()) - .map(|channel_ids| channel_ids.contains(&channel_id.to_string())) - .unwrap_or(false)) - } - - async fn remove_node_id( - &self, - uaid: &Uuid, - node_id: &str, - connected_at: u64, - _version: &Option, - ) -> DbResult { - let input = UpdateItemInput { - key: ddb_item! { uaid: s => uaid.simple().to_string() }, - update_expression: Some("REMOVE node_id".to_string()), - condition_expression: Some("(node_id = :node) and (connected_at = :conn)".to_string()), - expression_attribute_values: Some(hashmap! { - ":node".to_string() => val!(S => node_id), - ":conn".to_string() => val!(N => connected_at.to_string()) - }), - table_name: self.settings.router_table.clone(), - ..Default::default() - }; - - let result = retry_policy() - .retry_if( - || self.db_client.update_item(input.clone()), - retryable_updateitem_error(self.metrics.clone()), - ) - .await; - match result { - Ok(_) => Ok(true), - Err(RusotoError::Service(UpdateItemError::ConditionalCheckFailed(_))) => Ok(false), - Err(e) => Err(e.into()), - } - } - - async fn fetch_topic_messages( - &self, - uaid: &Uuid, - limit: usize, - ) -> DbResult { - // from commands::fetch_topic_messages() - let attr_values = hashmap! { - ":uaid".to_string() => val!(S => uaid.simple().to_string()), - ":cmi".to_string() => val!(S => "02"), - }; - let input = QueryInput { - key_condition_expression: Some("uaid = :uaid AND chidmessageid < :cmi".to_string()), - expression_attribute_values: Some(attr_values), - table_name: self.settings.message_table.to_string(), - consistent_read: Some(true), - limit: Some(limit as i64), - ..Default::default() - }; - - let output = retry_policy() - .retry_if( - || self.db_client.query(input.clone()), - retryable_query_error(self.metrics.clone()), - ) - .await?; - - let mut notifs: Vec = output.items.map_or_else(Vec::new, |items| { - debug!("Got response of: {:?}", items); - items - .into_iter() - .inspect(|i| debug!("Item: {:?}", i)) - .filter_map(|item| { - let item2 = item.clone(); - ok_or_inspect(serde_dynamodb::from_hashmap(item), |e| { - conversion_err(&self.metrics, e, item2, "serde_dynamodb_from_hashmap") - }) - }) - .collect() - }); - if notifs.is_empty() { - return Ok(Default::default()); - } - - // Load the current_timestamp from the subscription registry entry which is - // the first DynamoDbNotification and remove it from the vec. - let timestamp = notifs.remove(0).current_timestamp; - // Convert any remaining DynamoDbNotifications to Notification's - let messages = notifs - .into_iter() - .filter_map(|ddb_notif| { - let ddb_notif2 = ddb_notif.clone(); - ok_or_inspect(ddb_notif.into_notif(), |e| { - conversion_err(&self.metrics, e, ddb_notif2, "into_notif") - }) - }) - .collect(); - Ok(FetchMessageResponse { - timestamp, - messages, - }) - } - - async fn fetch_timestamp_messages( - &self, - uaid: &Uuid, - timestamp: Option, - limit: usize, - ) -> DbResult { - // Specify the minimum value to look for as a channelmessageid - let range_key = if let Some(ts) = timestamp { - format!("02:{}:z", ts) - } else { - // fun ascii tricks? because ':' comes before ';', look for any non-topic? - "01;".to_string() - }; - let attr_values = hashmap! { - ":uaid".to_string() => val!(S => uaid.simple().to_string()), - ":cmi".to_string() => val!(S => range_key), - }; - let input = QueryInput { - key_condition_expression: Some("uaid = :uaid AND chidmessageid > :cmi".to_string()), - expression_attribute_values: Some(attr_values), - table_name: self.settings.message_table.to_string(), - consistent_read: Some(true), - limit: Some(limit as i64), - ..Default::default() - }; - - let output = retry_policy() - .retry_if( - || self.db_client.query(input.clone()), - retryable_query_error(self.metrics.clone()), - ) - .await?; - - let messages = output.items.map_or_else(Vec::new, |items| { - debug!("Got response of: {:?}", items); - items - .into_iter() - .filter_map(|item| { - let item2 = item.clone(); - ok_or_inspect(serde_dynamodb::from_hashmap(item), |e| { - conversion_err(&self.metrics, e, item2, "serde_dynamodb_from_hashmap") - }) - }) - .filter_map(|ddb_notif: NotificationRecord| { - let ddb_notif2 = ddb_notif.clone(); - ok_or_inspect(ddb_notif.into_notif(), |e| { - conversion_err(&self.metrics, e, ddb_notif2, "into_notif") - }) - }) - .collect() - }); - let timestamp = messages.iter().filter_map(|m| m.sortkey_timestamp).max(); - Ok(FetchMessageResponse { - timestamp, - messages, - }) - } - - async fn increment_storage(&self, uaid: &Uuid, timestamp: u64) -> DbResult<()> { - let expiry = sec_since_epoch() + 2 * MAX_EXPIRY; - let attr_values = hashmap! { - ":timestamp".to_string() => val!(N => timestamp.to_string()), - ":expiry".to_string() => val!(N => expiry), - }; - let update_input = UpdateItemInput { - key: ddb_item! { - uaid: s => uaid.as_simple().to_string(), - chidmessageid: s => " ".to_string() - }, - update_expression: Some( - "SET current_timestamp = :timestamp, expiry = :expiry".to_string(), - ), - expression_attribute_values: Some(attr_values), - table_name: self.settings.message_table.clone(), - ..Default::default() - }; - - retry_policy() - .retry_if( - || self.db_client.update_item(update_input.clone()), - retryable_updateitem_error(self.metrics.clone()), - ) - .await?; - - Ok(()) - } - - async fn save_message(&self, uaid: &Uuid, message: Notification) -> DbResult<()> { - let topic = message.topic.is_some().to_string(); - let input = PutItemInput { - item: serde_dynamodb::to_hashmap(&NotificationRecord::from_notif(uaid, message))?, - table_name: self.settings.message_table.clone(), - ..Default::default() - }; - - retry_policy() - .retry_if( - || self.db_client.put_item(input.clone()), - retryable_putitem_error(self.metrics.clone()), - ) - .await?; - - self.metrics - .incr_with_tags("notification.message.stored") - .with_tag("topic", &topic) - .with_tag("database", &self.name()) - .send(); - Ok(()) - } - - async fn save_messages(&self, uaid: &Uuid, messages: Vec) -> DbResult<()> { - let put_items: Vec = messages - .into_iter() - .filter_map(|n| { - // eventually include `internal` if `meta` defined. - self.metrics - .incr_with_tags("notification.message.stored") - .with_tag("topic", &n.topic.is_some().to_string()) - .with_tag("database", &self.name()) - .send(); - serde_dynamodb::to_hashmap(&NotificationRecord::from_notif(uaid, n)) - .ok() - .map(|hm| WriteRequest { - put_request: Some(PutRequest { item: hm }), - delete_request: None, - }) - }) - .collect(); - let batch_input = BatchWriteItemInput { - request_items: hashmap! { self.settings.message_table.clone() => put_items }, - ..Default::default() - }; - - retry_policy() - .retry_if( - || self.db_client.batch_write_item(batch_input.clone()), - retryable_batchwriteitem_error(self.metrics.clone()), - ) - .await?; - Ok(()) - } - - async fn remove_message(&self, uaid: &Uuid, sort_key: &str) -> DbResult<()> { - let input = DeleteItemInput { - table_name: self.settings.message_table.clone(), - key: ddb_item! { - uaid: s => uaid.simple().to_string(), - chidmessageid: s => sort_key.to_owned() - }, - ..Default::default() - }; - - retry_policy() - .retry_if( - || self.db_client.delete_item(input.clone()), - retryable_delete_error(self.metrics.clone()), - ) - .await?; - self.metrics - .incr_with_tags("notification.message.deleted") - .with_tag("database", &self.name()) - .send(); - Ok(()) - } - - async fn router_table_exists(&self) -> DbResult { - self.table_exists(self.settings.router_table.clone()).await - } - - async fn message_table_exists(&self) -> DbResult { - self.table_exists(self.settings.message_table.clone()).await - } - - /// Perform a simple health check to make sure that the database is there. - /// This is called by __health__, so it should be reasonably light weight. - async fn health_check(&self) -> DbResult { - let input = ListTablesInput { - exclusive_start_table_name: Some(self.settings.message_table.clone()), - ..Default::default() - }; - // if we can't connect, that's a fail. - let result = self - .db_client - .list_tables(input) - .await - .map_err(|e| DbError::General(format!("DynamoDB health check failure: {:?}", e)))?; - if let Some(names) = result.table_names { - // We found at least one table that matches the message_table - debug!("dynamodb ok"); - return Ok(!names.is_empty()); - } - // Huh, we couldn't find a message table? That's a failure. - return Err(DbError::General( - "DynamoDB health check failure: No message table found".to_owned(), - )); - } - - fn box_clone(&self) -> Box { - Box::new(self.clone()) - } - - fn name(&self) -> String { - "DynamoDb".to_owned() - } -} diff --git a/autopush-common/src/db/dynamodb/retry.rs b/autopush-common/src/db/dynamodb/retry.rs deleted file mode 100644 index ffcc3807e..000000000 --- a/autopush-common/src/db/dynamodb/retry.rs +++ /dev/null @@ -1,61 +0,0 @@ -use again::RetryPolicy; -use cadence::{CountedExt, StatsdClient}; -use rusoto_core::RusotoError; -use rusoto_dynamodb::{ - BatchWriteItemError, DeleteItemError, DescribeTableError, GetItemError, PutItemError, - QueryError, UpdateItemError, -}; -use std::sync::Arc; - -/// Create a retry function for the given error -macro_rules! retryable_error { - ($name:ident, $error:tt, $error_tag:expr) => { - pub fn $name(metrics: Arc) -> impl Fn(&RusotoError<$error>) -> bool { - move |err| match err { - RusotoError::HttpDispatch(_) - | RusotoError::Service($error::InternalServerError(_)) - | RusotoError::Service($error::ProvisionedThroughputExceeded(_)) => { - debug!("retryable {} {:?}", $error_tag, &err); - metrics - .incr_with_tags("database.retry") - .with_tag("error", $error_tag) - .send(); - true - } - _ => false, - } - } - }; -} - -retryable_error!(retryable_query_error, QueryError, "query"); -retryable_error!(retryable_getitem_error, GetItemError, "get_item"); -retryable_error!(retryable_updateitem_error, UpdateItemError, "update_item"); -retryable_error!(retryable_putitem_error, PutItemError, "put_item"); -retryable_error!(retryable_delete_error, DeleteItemError, "delete_item"); -retryable_error!( - retryable_batchwriteitem_error, - BatchWriteItemError, - "batch_write_item" -); - -// DescribeTableError does not have a ProvisionedThroughputExceeded variant -pub fn retryable_describe_table_error( - metrics: Arc, -) -> impl Fn(&RusotoError) -> bool { - move |err| match err { - RusotoError::Service(DescribeTableError::InternalServerError(_)) => { - metrics - .incr_with_tags("database.retry") - .with_tag("error", "describe_table_error") - .send(); - true - } - _ => false, - } -} - -/// Build an exponential retry policy -pub fn retry_policy() -> RetryPolicy { - RetryPolicy::default().with_jitter(true) -} diff --git a/autopush-common/src/db/error.rs b/autopush-common/src/db/error.rs index b25284417..a30dc915b 100644 --- a/autopush-common/src/db/error.rs +++ b/autopush-common/src/db/error.rs @@ -1,12 +1,5 @@ use actix_web::http::StatusCode; -#[cfg(feature = "dynamodb")] -use rusoto_core::RusotoError; -#[cfg(feature = "dynamodb")] -use rusoto_dynamodb::{ - BatchWriteItemError, DeleteItemError, DescribeTableError, GetItemError, PutItemError, - QueryError, UpdateItemError, -}; use thiserror::Error; #[cfg(feature = "bigtable")] @@ -17,38 +10,6 @@ pub type DbResult = Result; #[derive(Debug, Error)] pub enum DbError { - #[cfg(feature = "dynamodb")] - #[error("Database error while performing GetItem")] - DdbGetItem(#[from] RusotoError), - - #[cfg(feature = "dynamodb")] - #[error("Database error while performing UpdateItem")] - DdbUpdateItem(#[from] RusotoError), - - #[cfg(feature = "dynamodb")] - #[error("Database error while performing PutItem")] - DdbPutItem(#[from] RusotoError), - - #[cfg(feature = "dynamodb")] - #[error("Database error while performing DeleteItem")] - DdbDeleteItem(#[from] RusotoError), - - #[cfg(feature = "dynamodb")] - #[error("Database error while performing BatchWriteItem")] - DdbBatchWriteItem(#[from] RusotoError), - - #[cfg(feature = "dynamodb")] - #[error("Database error while performing DescribeTable")] - DdbDescribeTable(#[from] RusotoError), - - #[cfg(feature = "dynamodb")] - #[error("Database error while performing Query")] - DdbQuery(#[from] RusotoError), - - #[cfg(feature = "dynamodb")] - #[error("Error while performing DynamoDB (de)serialization: {0}")] - DdbSerialization(#[from] serde_dynamodb::Error), - #[error("Error while performing (de)serialization: {0}")] Serialization(String), diff --git a/autopush-common/src/db/mod.rs b/autopush-common/src/db/mod.rs index 0f014cb94..4624885aa 100644 --- a/autopush-common/src/db/mod.rs +++ b/autopush-common/src/db/mod.rs @@ -21,8 +21,6 @@ use uuid::Uuid; #[cfg(feature = "bigtable")] pub mod bigtable; pub mod client; -#[cfg(feature = "dynamodb")] -pub mod dynamodb; pub mod error; pub mod models; pub mod reporter; @@ -50,8 +48,6 @@ pub enum StorageType { INVALID, #[cfg(feature = "bigtable")] BigTable, - #[cfg(feature = "dynamodb")] - DynamoDb, } impl From<&str> for StorageType { @@ -59,8 +55,6 @@ impl From<&str> for StorageType { match name.to_lowercase().as_str() { #[cfg(feature = "bigtable")] "bigtable" => Self::BigTable, - #[cfg(feature = "dynamodb")] - "dynamodb" => Self::DynamoDb, _ => Self::INVALID, } } @@ -72,8 +66,6 @@ impl StorageType { fn available<'a>() -> Vec<&'a str> { #[allow(unused_mut)] let mut result: Vec<&str> = Vec::new(); - #[cfg(feature = "dynamodb")] - result.push("DynamoDB"); #[cfg(feature = "bigtable")] result.push("Bigtable"); result @@ -87,14 +79,7 @@ impl StorageType { info!("No DSN specified, failing over to old default dsn: {default}"); return Self::from(default); } - let dsn = dsn - .clone() - .unwrap_or(std::env::var("AWS_LOCAL_DYNAMODB").unwrap_or_default()); - #[cfg(feature = "dynamodb")] - if dsn.starts_with("http") { - trace!("Found http"); - return Self::DynamoDb; - } + let dsn = dsn.clone().unwrap_or_default(); #[cfg(feature = "bigtable")] if dsn.starts_with("grpc") { trace!("Found grpc"); @@ -120,9 +105,8 @@ pub struct DbSettings { pub dsn: Option, /// A JSON formatted dictionary containing Database settings that /// are specific to the type of Data storage specified in the `dsn` - /// See the respective settings structures for - /// [crate::db::dynamodb::DynamoDbSettings] - /// and [crate::db::bigtable::BigTableDbSettings] + /// See the respective settings structure for + /// [crate::db::bigtable::BigTableDbSettings] pub db_settings: String, } //TODO: add `From for DbSettings`? @@ -155,7 +139,6 @@ pub struct CheckStorageResponse { pub struct User { /// The UAID. This is generally a UUID4. It needs to be globally /// unique. - // DynamoDB #[serde(serialize_with = "uuid_serializer")] pub uaid: Uuid, /// Time in milliseconds that the user last connected at @@ -203,10 +186,8 @@ impl Default for User { #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct NotificationRecord { /// The UserAgent Identifier (UAID) - // DynamoDB #[serde(serialize_with = "uuid_serializer")] uaid: Uuid, - // DynamoDB // Format: // Topic Messages: // {TOPIC_NOTIFICATION_PREFIX}:{channel id}:{topic} @@ -224,8 +205,7 @@ pub struct NotificationRecord { /// Time in seconds from epoch #[serde(skip_serializing_if = "Option::is_none")] timestamp: Option, - /// DynamoDB expiration timestamp per - /// + /// Expiration timestamp expiry: u64, /// TTL value provided by application server for the message #[serde(skip_serializing_if = "Option::is_none")] diff --git a/autopush-common/src/db/routing.rs b/autopush-common/src/db/routing.rs index 6cbda70a5..7999552aa 100644 --- a/autopush-common/src/db/routing.rs +++ b/autopush-common/src/db/routing.rs @@ -1,15 +1,12 @@ #[derive(Clone, Eq, PartialEq, Debug)] pub(crate) enum StorageType { - DynamoDB, BigTable, None, } impl Default for StorageType { fn default() -> StorageType { - if cfg!(feature = "dynamodb") { - StorageType::DynamoDB - } else if cfg!(feature = "bigtable") { + if cfg!(feature = "bigtable") { StorageType::BigTable } else { StorageType::None @@ -20,7 +17,6 @@ impl Default for StorageType { impl From<&str> for StorageType { fn from(str: &str) -> StorageType { match str.to_lowercase().as_str() { - "dynamodb" => StorageType::DynamoDB, "bigtable" => StorageType::BigTable, _ => { warn!("Using default StorageType for {str}"); diff --git a/configs/autopush.toml.sample b/configs/autopush.toml.sample index 67a8fc32c..02a3c98d1 100644 --- a/configs/autopush.toml.sample +++ b/configs/autopush.toml.sample @@ -51,10 +51,6 @@ # The port of the metrics server #statsd_port = 8125 -# Override the DynamoDB endpoint via the AWS_LOCAL_DYNAMODB environment -# variable. No default value. -#aws_ddb_endpoint = "..." - # The name of the router table #router_tablename = "router" diff --git a/docs/src/architecture.md b/docs/src/architecture.md index 0430cb96d..7bef278e0 100644 --- a/docs/src/architecture.md +++ b/docs/src/architecture.md @@ -216,17 +216,17 @@ Autopush used a [table rotation system](table_rotation.md), which is now legacy. Endpoint will deliver the message to the client completely bypassing Storage. This Notification will be referred to as a Direct Notification vs. a Stored Notification. -* (_DynamoDb_) Provisioned Write Throughput for the Router table determines how +* (_DynamoDb (legacy)_) Provisioned Write Throughput for the Router table determines how many connections per second can be accepted across the entire cluster. -* (_DynamoDb_) Provisioned Read Throughput for the Router table **and** Provisioned +* (_DynamoDb (legacy)_) Provisioned Read Throughput for the Router table **and** Provisioned Write throughput for the Storage table determine maximum possible notifications per second that can be handled. In theory notification throughput can be higher than Provisioned Write Throughput on the Storage as connected clients will frequently not require using Storage at all. Read's to the Router table are still needed for every notification, whether Storage is hit or not. -* (_DynamoDb_) Provisioned Read Throughput on for the Storage table is an important +* (_DynamoDb (legacy)_) Provisioned Read Throughput on for the Storage table is an important factor in maximum notification throughput, as many slow clients may require frequent Storage checks. * If a client is reconnecting, their Router record will be old. Router