Merge branch 'master' into release/0.5

mozilla-services · Aug 22, 2020 · af41520 · af41520
2 parents 1da1d20 + 0a2fecc
commit af41520
Show file tree

Hide file tree

Showing 42 changed files with 632 additions and 270 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,38 @@
+<a name="0.5.6"></a>
+## 0.5.6 (2020-08-11)
+
+
+#### Features
+
+*   More purge_ttl features (#776) ([59aa28a4](https://github.com/mozilla-services/syncstorage-rs/commit/59aa28a4e5fdcfe2acc3f767487066d30b998af0), closes [#735](https://github.com/mozilla-services/syncstorage-rs/issues/735), [#743](https://github.com/mozilla-services/syncstorage-rs/issues/743))
+
+#### Bug Fixes
+
+*   remove ubuntu target for grpcio (#775) ([7d1061f7](https://github.com/mozilla-services/syncstorage-rs/commit/7d1061f7197a56936a6cff9a438997640892d6c6), closes [#774](https://github.com/mozilla-services/syncstorage-rs/issues/774))
+*   Return WeaveError::OverQuota for over quota responses (#773) ([38cd5ddd](https://github.com/mozilla-services/syncstorage-rs/commit/38cd5dddc36ae0aeda159fea88ba6128a8e85181), closes [#769](https://github.com/mozilla-services/syncstorage-rs/issues/769))
+*   ensure an X-Last-Modified for /info/configuration (#761) ([36533f85](https://github.com/mozilla-services/syncstorage-rs/commit/36533f8566c39e8c82ccb5a2bc8ae62fb254129a), closes [#759](https://github.com/mozilla-services/syncstorage-rs/issues/759))
+
+
+
+<a name="0.5.5"></a>
+### 0.5.5 (2020-08-06)
+
+#### Bug Fixes
+
+*   set config env separator to double underscore. (#763) ([f1d88fea](https://github.com/mozilla-services/syncstorage-rs/commit/f1d88feae60d7fea15b7575ac2108f0f80ff42b4), closes [#762](https://github.com/mozilla-services/syncstorage-rs/issues/762))
+
+
+
+<a name="0.5.4"></a>
+### 0.5.4 (2020-08-04)
+
+
+#### Features
+
+*   add debug_client check to BsoBodies for batch operations. ([1370df9d](https://github.com/mozilla-services/syncstorage-rs/commit/1370df9d7c2e6d656f50332b3f8615faafacead0)
+
+
+
 <a name="0.5.3"></a>
 ## 0.5.3 (2020-07-31)
 

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "syncstorage"
-version = "0.5.3"
+version = "0.5.5"
 license = "MPL-2.0"
 authors = [
   "Ben Bangert <[email protected]>",
@@ -26,6 +26,7 @@ bytes = "0.5"
 cadence = "0.20.0"
 chrono = "0.4"
 config = "0.10"
+deadpool = "0.5.2"
 diesel = { version = "1.4.4", features = ["mysql", "r2d2"] }
 diesel_logger = "0.1.1"
 diesel_migrations = { version = "1.4.0", features = ["mysql"] }
@@ -34,6 +35,9 @@ env_logger = "0.7.1"
 failure = "0.1.8"
 futures = { version = "0.3", features = ["compat"] }
 googleapis-raw = { version = "0", path = "vendor/mozilla-rust-sdk/googleapis-raw" }
+# Some versions of OpenSSL 1.1.1 conflict with grpcio's built-in boringssl which can cause
+# syncserver to either fail to either compile, or start. In those cases, try
+# `cargo build --features grpcio/openssl ...`
 grpcio = { version = "0.6.0" }
 lazy_static = "1.4.0"
 hawk = "3.2"
@@ -44,7 +48,7 @@ log = { version = "0.4.8", features = ["max_level_info", "release_max_level_info
 mime = "0.3"
 num_cpus = "1"
 # must match what's used by googleapis-raw
-protobuf = "2.15"
+protobuf = "2.17.0"
 rand = "0.7"
 regex = "1.3"
 sentry = { version = "0.19", features = ["with_curl_transport"] }
@@ -62,6 +66,7 @@ slog-stdlog = "4.0"
 slog-term = "2.6"
 time = "0.2"
 url = "2.1"
+urlencoding = "1.1"
 uuid = { version = "0.8.1", features = ["serde", "v4"] }
 validator = "0.10"
 validator_derive = "0.10"

diff --git a/README.md b/README.md
@@ -122,7 +122,7 @@ This requires access to the mozilla-rust-sdk which is now available at `/vendor/
 
 ### Connecting to Firefox
 
-This will walk you through the steps to connect this project to your local copy of Firefox. 
+This will walk you through the steps to connect this project to your local copy of Firefox.
 
 1. Follow the steps outlined above for running this project using [MySQL](https://github.com/mozilla-services/syncstorage-rs#mysql).
 
@@ -201,11 +201,13 @@ Open a PR after doing the following:
 
 Once your PR merges, then go ahead and create an official [GitHub release](https://github.com/mozilla-services/syncstorage-rs/releases).
 
-
 ## Troubleshooting
 
 - `rm Cargo.lock; cargo clean;` - Try this if you're having problems compiling.
 
+- Some versions of OpenSSL 1.1.1 can conflict with grpcio's built in BoringSSL. These errors can cause syncstorage to fail to run or compile.
+If you see a problem related to `libssl` you may need to specify the `cargo` option `--features grpcio/openssl`  to force grpcio to use OpenSSL.
+
 ## Related Documentation
 
 - [API docs](https://mozilla-services.readthedocs.io/en/latest/storage/apis-1.5.html)

diff --git a/src/db/results.rs b/src/db/results.rs
@@ -94,6 +94,15 @@ impl From<bb8::State> for PoolState {
     }
 }
 
+impl From<deadpool::Status> for PoolState {
+    fn from(status: deadpool::Status) -> PoolState {
+        PoolState {
+            connections: status.size as u32,
+            idle_connections: status.available as u32,
+        }
+    }
+}
+
 #[cfg(test)]
 pub type GetCollectionId = i32;
 

diff --git a/src/db/spanner/batch.rs b/src/db/spanner/batch.rs
@@ -18,7 +18,7 @@ use crate::{
 };
 
 pub async fn create_async(
-    db: &SpannerDb<'_>,
+    db: &SpannerDb,
     params: params::CreateBatch,
 ) -> Result<results::CreateBatch> {
     let batch_id = Uuid::new_v4().to_simple().to_string();
@@ -57,7 +57,7 @@ pub async fn create_async(
     Ok(batch_id)
 }
 
-pub async fn validate_async(db: &SpannerDb<'_>, params: params::ValidateBatch) -> Result<bool> {
+pub async fn validate_async(db: &SpannerDb, params: params::ValidateBatch) -> Result<bool> {
     let collection_id = db.get_collection_id_async(&params.collection).await?;
     let exists = db
         .sql(
@@ -81,7 +81,7 @@ pub async fn validate_async(db: &SpannerDb<'_>, params: params::ValidateBatch) -
     Ok(exists.is_some())
 }
 
-pub async fn append_async(db: &SpannerDb<'_>, params: params::AppendToBatch) -> Result<()> {
+pub async fn append_async(db: &SpannerDb, params: params::AppendToBatch) -> Result<()> {
     let mut metrics = db.metrics.clone();
     metrics.start_timer("storage.spanner.append_items_to_batch", None);
 
@@ -106,7 +106,7 @@ pub async fn append_async(db: &SpannerDb<'_>, params: params::AppendToBatch) ->
 }
 
 pub async fn get_async(
-    db: &SpannerDb<'_>,
+    db: &SpannerDb,
     params: params::GetBatch,
 ) -> Result<Option<results::GetBatch>> {
     let collection_id = db.get_collection_id_async(&params.collection).await?;
@@ -142,7 +142,7 @@ pub async fn get_async(
     Ok(batch)
 }
 
-pub async fn delete_async(db: &SpannerDb<'_>, params: params::DeleteBatch) -> Result<()> {
+pub async fn delete_async(db: &SpannerDb, params: params::DeleteBatch) -> Result<()> {
     let collection_id = db.get_collection_id_async(&params.collection).await?;
     // Also deletes child batch_bsos rows (INTERLEAVE IN PARENT batches ON
     // DELETE CASCADE)
@@ -165,7 +165,7 @@ pub async fn delete_async(db: &SpannerDb<'_>, params: params::DeleteBatch) -> Re
 }
 
 pub async fn commit_async(
-    db: &SpannerDb<'_>,
+    db: &SpannerDb,
     params: params::CommitBatch,
 ) -> Result<results::CommitBatch> {
     let mut metrics = db.metrics.clone();
@@ -239,7 +239,7 @@ pub async fn commit_async(
 }
 
 pub async fn do_append_async(
-    db: &SpannerDb<'_>,
+    db: &SpannerDb,
     user_id: HawkIdentifier,
     collection_id: i32,
     batch_id: String,
@@ -335,7 +335,7 @@ pub async fn do_append_async(
 /// For the special case of a user creating a batch for a collection with no
 /// prior data.
 async fn pretouch_collection_async(
-    db: &SpannerDb<'_>,
+    db: &SpannerDb,
     user_id: &HawkIdentifier,
     collection_id: i32,
 ) -> Result<()> {

diff --git a/src/db/spanner/manager.rs → src/db/spanner/manager/bb8.rs b/src/db/spanner/manager.rs → src/db/spanner/manager/bb8.rs
@@ -18,7 +18,7 @@ use crate::{
     settings::Settings,
 };
 
-const SPANNER_ADDRESS: &str = "spanner.googleapis.com:443";
+pub const SPANNER_ADDRESS: &str = "spanner.googleapis.com:443";
 
 pub struct SpannerConnectionManager<T> {
     database_name: String,
@@ -33,6 +33,7 @@ impl<_T> fmt::Debug for SpannerConnectionManager<_T> {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt.debug_struct("SpannerConnectionManager")
             .field("database_name", &self.database_name)
+            .field("test_transactions", &self.test_transactions)
             .finish()
     }
 }
@@ -65,7 +66,7 @@ pub struct SpannerSession {
     pub client: SpannerClient,
     pub session: Session,
 
-    pub(super) use_test_transactions: bool,
+    pub(in crate::db::spanner) use_test_transactions: bool,
 }
 
 #[async_trait]
@@ -130,7 +131,7 @@ impl<T: std::marker::Send + std::marker::Sync + 'static> ManageConnection
     }
 }
 
-async fn create_session(
+pub async fn create_session(
     client: &SpannerClient,
     database_name: &str,
 ) -> Result<Session, grpcio::Error> {