refactor: add common crates (#1281)

Closes #1275

.circleci/config.yml

@@ -43,7 +43,7 @@ commands:
       - run:
           name: Core Python Checks
           command: |
-            flake8 src/tokenserver
+            flake8 syncstorage/src/tokenserver
             flake8 tools/integration_tests
             flake8 tools/tokenserver
   rust-clippy:

Cargo.toml

@@ -1,94 +1,13 @@
-[package]
-name = "syncstorage"
-version = "0.10.2"
-license = "MPL-2.0"
-authors = [
-  "Ben Bangert <[email protected]>",
-  "Phil Jenvey <[email protected]>",
-  "Mozilla Services Engineering <[email protected]>"
+[workspace]
+resolver = "2"
+members = [
+    "syncstorage-db-common",
+    "syncstorage-common",
+    "tokenserver-common",
+    "syncstorage",
 ]
-edition = "2018"
-default-run = "syncstorage"
+default-members = ["syncstorage"]
 
 [profile.release]
-# Enables line numbers in Sentry
+# Enables line numbers in Sentry reporting
 debug = 1
-
-[dependencies]
-actix-http = "2"
-actix-web = "3"
-actix-web-httpauth = "0.5"
-actix-rt = "1"  # Pin to 1.0, due to dependencies on Tokio
-actix-cors = "0.5"
-async-trait = "0.1.40"
-backtrace = "0.3.61"
-base64 = "0.13"
-bb8 = "0.4.1"  # pin to 0.4 due to dependencies on Tokio
-bytes = "1.0"
-cadence = "0.26"
-chrono = "0.4"
-config = "0.11"
-# Pin to 0.5 for now, to keep it under tokio 0.2 (issue977).
-# Fix for #803 (deadpool#92) points to our fork for now
-#deadpool = "0.5"  # pin to 0.5
-deadpool = { git = "https://github.com/mozilla-services/deadpool", branch = "deadpool-v0.5.2-issue92" }
-diesel = { version = "1.4", features = ["mysql", "r2d2"] }
-diesel_logger = "0.1.1"
-diesel_migrations = { version = "1.4.0", features = ["mysql"] }
-docopt = "1.1.0"
-dyn-clone = "1.0.4"
-env_logger = "0.9"
-futures = { version = "0.3", features = ["compat"] }
-google-cloud-rust-raw = "0.11.0"
-# Some versions of OpenSSL 1.1.1 conflict with grpcio's built-in boringssl which can cause
-# syncserver to either fail to either compile, or start. In those cases, try
-# `cargo build --features grpcio/openssl ...`
-grpcio = { version = "0.9" }
-lazy_static = "1.4.0"
-hawk = "3.2"
-hex = "0.4.3"
-hostname = "0.3.1"
-hkdf = "0.11"
-hmac = "0.11"
-http = "0.2.5"
-log = { version = "0.4", features = ["max_level_debug", "release_max_level_info"] }
-mime = "0.3"
-mockito = "0.30.0"
-num_cpus = "1"
-# must match what's used by googleapis-raw
-protobuf = "2.20.0"
-pyo3 = { version = "0.14", features = ["auto-initialize"] }
-rand = "0.8"
-regex = "1.4"
-reqwest = { version = "0.10.10", features = ["json", "rustls-tls"] }
-# pin to 0.19: https://github.com/getsentry/sentry-rust/issues/277
-sentry = { version = "0.19", features = ["with_curl_transport"] }# pin to 0.19 until on-prem sentry server is updated
-sentry-backtrace = "0.19"
-serde = "1.0"
-serde_derive = "1.0"
-serde_json = { version = "1.0", features = ["arbitrary_precision"] }
-scheduled-thread-pool = "0.2"
-sha2 = "0.9"
-slog = { version = "2.5", features = ["max_level_info", "release_max_level_info", "dynamic-keys"] }
-slog-async = "2.5"
-slog-envlogger = "2.2.0"
-slog-mozlog-json = "0.1"
-slog-scope = "4.3"
-slog-stdlog = "4.1"
-slog-term = "2.6"
-thiserror = "1.0.26"
-time = "^0.3"
-# pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.)
-tokio = { version = "0.2.4", features = ["macros", "sync"] }
-url = "2.1"
-urlencoding = "2.1"
-uuid = { version = "0.8.2", features = ["serde", "v4"] }
-validator = "0.14"
-validator_derive = "0.14"
-woothee = "0.11"
-
-[features]
-no_auth = []
-
-[[bin]]
-name = "purge_ttl"

Dockerfile

@@ -11,8 +11,8 @@ RUN apt-get -q update && \
 RUN \
     cargo --version && \
     rustc --version && \
-    cargo install --path . --locked --root /app && \
-    cargo install --path . --locked --root /app --bin purge_ttl
+    cargo install --path ./syncstorage --locked --root /app && \
+    cargo install --path ./syncstorage --locked --root /app --bin purge_ttl
 
 FROM debian:buster-slim
 WORKDIR /app
@@ -40,7 +40,7 @@ COPY --from=builder /app/spanner_config.ini /app
 COPY --from=builder /app/tools/spanner /app/tools/spanner
 COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests
 COPY --from=builder /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh
-COPY --from=builder /app/src/db/spanner/schema.ddl /app/schema.ddl
+COPY --from=builder /app/syncstorage/src/db/spanner/schema.ddl /app/schema.ddl
 
 RUN chmod +x /app/scripts/prepare-spanner.sh
 RUN pip3 install -r /app/tools/integration_tests/requirements.txt

syncstorage-common/Cargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "syncstorage-common"
+version = "0.10.2"
+edition = "2021"
+
+[dependencies]

syncstorage-common/src/lib.rs

@@ -0,0 +1,21 @@
+#[macro_export]
+macro_rules! from_error {
+    ($from:ty, $to:ty, $to_kind:expr) => {
+        impl From<$from> for $to {
+            fn from(inner: $from) -> $to {
+                $to_kind(inner).into()
+            }
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! impl_fmt_display {
+    ($error:ty, $kind:ty) => {
+        impl fmt::Display for $error {
+            fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+                fmt::Display::fmt(&self.kind, formatter)
+            }
+        }
+    };
+}

syncstorage-db-common/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "syncstorage-db-common"
+version = "0.10.2"
+edition = "2021"
+
+[dependencies]
+async-trait = "0.1.40"
+backtrace = "0.3.61"
+chrono = "0.4"
+# Pin to 0.5 for now, to keep it under tokio 0.2 (issue977).
+# Fix for #803 (deadpool#92) points to our fork for now
+#deadpool = "0.5"  # pin to 0.5
+deadpool = { git = "https://github.com/mozilla-services/deadpool", branch = "deadpool-v0.5.2-issue92" }
+diesel = { version = "1.4", features = ["mysql", "r2d2"] }
+diesel_migrations = { version = "1.4.0", features = ["mysql"] }
+# Some versions of OpenSSL 1.1.1 conflict with grpcio's built-in boringssl which can cause
+# syncserver to either fail to either compile, or start. In those cases, try
+# `cargo build --features grpcio/openssl ...`
+grpcio = { version = "0.9" }
+hostname = "0.3.1"
+http = "0.2.6"
+futures = { version = "0.3", features = ["compat"] }
+lazy_static = "1.4.0"
+serde = "1.0"
+serde_json = { version = "1.0", features = ["arbitrary_precision"] }
+syncstorage-common = { path = "../syncstorage-common" }
+thiserror = "1.0.26"

src/db/error.rs → syncstorage-db-common/src/error.rs

@@ -1,12 +1,15 @@
 use std::fmt;
 
-use actix_web::http::StatusCode;
+use backtrace::Backtrace;
+use http::StatusCode;
+use syncstorage_common::{from_error, impl_fmt_display};
 use thiserror::Error;
 
-#[derive(Error, Debug)]
+#[derive(Debug)]
 pub struct DbError {
     kind: DbErrorKind,
     pub status: StatusCode,
+    pub backtrace: Backtrace,
 }
 
 #[derive(Debug, Error)]
@@ -38,9 +41,6 @@ pub enum DbErrorKind {
     #[error("Specified batch does not exist")]
     BatchNotFound,
 
-    #[error("Tokenserver user retired")]
-    TokenserverUserRetired,
-
     #[error("An attempt at a conflicting write")]
     Conflict,
 
@@ -61,10 +61,6 @@ pub enum DbErrorKind {
 }
 
 impl DbError {
-    pub fn kind(&self) -> &DbErrorKind {
-        &self.kind
-    }
-
     pub fn internal(msg: &str) -> Self {
         DbErrorKind::Internal(msg.to_owned()).into()
     }
@@ -79,6 +75,26 @@ impl DbError {
             _ => None,
         }
     }
+
+    pub fn is_collection_not_found(&self) -> bool {
+        matches!(self.kind, DbErrorKind::CollectionNotFound)
+    }
+
+    pub fn is_conflict(&self) -> bool {
+        matches!(self.kind, DbErrorKind::Conflict)
+    }
+
+    pub fn is_quota(&self) -> bool {
+        matches!(self.kind, DbErrorKind::Quota)
+    }
+
+    pub fn is_bso_not_found(&self) -> bool {
+        matches!(self.kind, DbErrorKind::BsoNotFound)
+    }
+
+    pub fn is_batch_not_found(&self) -> bool {
+        matches!(self.kind, DbErrorKind::BatchNotFound)
+    }
 }
 
 impl From<DbErrorKind> for DbError {
@@ -94,12 +110,14 @@ impl From<DbErrorKind> for DbError {
             //  * android bug: https://bugzilla.mozilla.org/show_bug.cgi?id=959032
             DbErrorKind::Conflict => StatusCode::SERVICE_UNAVAILABLE,
             DbErrorKind::Quota => StatusCode::FORBIDDEN,
-            // NOTE: TokenserverUserRetired is an internal service error for compatibility reasons
-            // (the legacy Tokenserver returned an internal service error in this situation)
             _ => StatusCode::INTERNAL_SERVER_ERROR,
         };
 
-        Self { kind, status }
+        Self {
+            kind,
+            status,
+            backtrace: Backtrace::new(),
+        }
     }
 }