feat: Add sqlite as a storage backend

First throw at adding sqlite as storage backend for the syncserver and the tokenserver. There is probably some duplicated code between: - syncstorage-mysql and syncstorage-sqlite - tokenserver-db-mysql and tokenserver-db-sqlite tokenserver-db-sqlite probably contains Mysql-specific SQL that might break Sqlite when ran. Squashed commit of the following: commit 1047197 Author: Eragon <[email protected]> Date: Wed Mar 6 00:24:07 2024 +0100 fix: Fix default for tokenserver-db commit 5e2a745 Author: Eragon <[email protected]> Date: Sun Feb 11 22:00:46 2024 +0100 tokenserver-db defaults to use mysql as db backend commit a587787 Author: Eragon <[email protected]> Date: Sun Feb 11 21:47:55 2024 +0100 Run cargo fmt commit c8c1458 Author: Eragon <[email protected]> Date: Sun Feb 11 20:32:19 2024 +0100 Better logging of migrations commit e2b8563 Author: Eragon <[email protected]> Date: Tue Jan 30 00:36:11 2024 +0100 wip: At least it runs now commit bd24d7c Author: Eragon <[email protected]> Date: Mon Jan 22 11:20:38 2024 +0100 lll commit 07ba38f Author: Eragon <[email protected]> Date: Fri Jan 19 14:35:20 2024 +0100 wip: First throw at adding sqlite as a storage backend
mozilla-services · Mar 10, 2024 · 9d83956 · 9d83956
1 parent 8faf728
commit 9d83956
Show file tree

Hide file tree

Showing 58 changed files with 4,888 additions and 264 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -9,9 +9,13 @@ members = [
   "syncstorage-mysql",
   "syncstorage-settings",
   "syncstorage-spanner",
+  "syncstorage-sqlite",
   "tokenserver-auth",
   "tokenserver-common",
   "tokenserver-db",
+  "tokenserver-db-common",
+  "tokenserver-db-mysql",
+  "tokenserver-db-sqlite",
   "tokenserver-settings",
   "syncserver",
 ]
@@ -23,6 +27,7 @@ authors = [
   "Ben Bangert <[email protected]>",
   "Phil Jenvey <[email protected]>",
   "Mozilla Services Engineering <[email protected]>",
+  "Eragon <[email protected]>",
 ]
 edition = "2021"
 license = "MPL-2.0"

diff --git a/Makefile b/Makefile
@@ -13,13 +13,17 @@ PATH_TO_GRPC_CERT = ../server-syncstorage/local/lib/python2.7/site-packages/grpc
 SRC_ROOT = $(shell pwd)
 PYTHON_SITE_PACKGES = $(shell $(SRC_ROOT)/venv/bin/python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
 
+clippy_sqlite:
+	# Matches what's run in circleci
+	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/sqlite,tokenserver-db/sqlite --features=py_verifier -- -D warnings
+
 clippy_mysql:
 	# Matches what's run in circleci
-	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/mysql --features=py_verifier -- -D warnings
+	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/mysql,tokenserver-db/mysql --features=py_verifier -- -D warnings
 
 clippy_spanner:
 	# Matches what's run in circleci
-	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/spanner --features=py_verifier -- -D warnings
+	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/spanner,tokenserver-db/mysql --features=py_verifier  -- -D warnings
 
 clean:
 	cargo clean
@@ -53,23 +57,38 @@ run_mysql: python
 		# See https://github.com/PyO3/pyo3/issues/1741 for discussion re: why we need to set the
 		# below env var
 		PYTHONPATH=$(PYTHON_SITE_PACKGES) \
-	        RUST_LOG=debug \
+		RUST_LOG=debug \
 		RUST_BACKTRACE=full \
-		cargo run --no-default-features --features=syncstorage-db/mysql --features=py_verifier -- --config config/local.toml
+		cargo run --no-default-features --features=syncstorage-db/mysql,tokenserver-db/mysql --features=py_verifier -- --config config/local.toml
+
+run_sqlite: python
+	PATH="./venv/bin:$(PATH)" \
+		# See https://github.com/PyO3/pyo3/issues/1741 for discussion re: why we need to set the
+		# below env var
+		PYTHONPATH=$(PYTHON_SITE_PACKGES) \
+		RUST_LOG=debug \
+		RUST_BACKTRACE=full \
+		cargo run --no-default-features --features=syncstorage-db/sqlite,tokenserver-db/sqlite --features=py_verifier -- --config config/local.toml
 
 run_spanner: python
 	GOOGLE_APPLICATION_CREDENTIALS=$(PATH_TO_SYNC_SPANNER_KEYS) \
 		GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$(PATH_TO_GRPC_CERT) \
 		# See https://github.com/PyO3/pyo3/issues/1741 for discussion re: why we need to set the
 		# below env var
 		PYTHONPATH=$(PYTHON_SITE_PACKGES) \
-	    PATH="./venv/bin:$(PATH)" \
+		PATH="./venv/bin:$(PATH)" \
 		RUST_LOG=debug \
 		RUST_BACKTRACE=full \
-		cargo run --no-default-features --features=syncstorage-db/spanner --features=py_verifier -- --config config/local.toml
+		cargo run --no-default-features --features=syncstorage-db/spanner,tokenserver-db/mysql --features=py_verifier -- --config config/local.toml
 
-test:
+test_mysql:
 	SYNC_SYNCSTORAGE__DATABASE_URL=mysql://sample_user:sample_password@localhost/syncstorage_rs \
 		SYNC_TOKENSERVER__DATABASE_URL=mysql://sample_user:sample_password@localhost/tokenserver_rs \
 		RUST_TEST_THREADS=1 \
 		cargo test --workspace
+
+test_sqlite:
+	SYNC_SYNCSTORAGE__DATABASE_URL=:memory: \
+		SYNC_TOKENSERVER__DATABASE_URL=:memory: \
+		RUST_TEST_THREADS=1 \
+		cargo test --workspace
diff --git a/README.md b/README.md
@@ -12,6 +12,7 @@ Mozilla Sync Storage built with [Rust](https://rust-lang.org).
 - [Local Setup](#local-setup)
   - [MySQL](#mysql)
   - [Spanner](#spanner)
+  - [Sqlite](#sqlite)
   - [Running via Docker](#running-via-docker)
   - [Connecting to Firefox](#connecting-to-firefox)
 - [Logging](#logging)
@@ -179,6 +180,15 @@ To run an application server that points to the local Spanner emulator:
 SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST=localhost:9010 make run_spanner
 ```
 
+### Sqlite
+
+Setting up the server with sqlite only requires a path to the database file,
+which will be created automatically:
+
+`sqlite:path/syncdb.sqlite`
+
+This requires at least sqlite v3.24.0 to be installed on the host system.
+
 ### Running via Docker
 
 This requires access to [Google Cloud Rust (raw)](https://crates.io/crates/google-cloud-rust-raw/) crate. Please note that due to interdependencies, you will need to ensure that `grpcio` and `protobuf` match the version used by `google-cloud-rust-raw`.

diff --git a/config/local.example.toml b/config/local.example.toml
@@ -5,9 +5,13 @@ human_logs = 1
 
 # Example Syncstorage settings:
 # Example MySQL DSN:
-syncstorage.database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs"
+#syncstorage.database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs"
 # Example Spanner DSN:
 # database_url="spanner://projects/SAMPLE_GCP_PROJECT/instances/SAMPLE_SPANNER_INSTANCE/databases/SAMPLE_SPANNER_DB"
+# Example SQLite DSN:
+# database_url="sqlite://PATH_TO_FILE/FILE.sqlite"
+syncstorage.database_url = ":memory:"
+
 # enable quota limits
 syncstorage.enable_quota = 0
 # set the quota limit to 2GB.
@@ -16,7 +20,8 @@ syncstorage.enabled = true
 syncstorage.limits.max_total_records = 1666 # See issues #298/#333
 
 # Example Tokenserver settings:
-tokenserver.database_url = "mysql://sample_user:sample_password@localhost/tokenserver_rs"
+#tokenserver.database_url = "mysql://sample_user:sample_password@localhost/tokenserver_rs"
+tokenserver.database_url = ":memory:"
 tokenserver.enabled = true
 tokenserver.fxa_email_domain = "api-accounts.stage.mozaws.net"
 tokenserver.fxa_metrics_hash_secret = "INSERT_SECRET_KEY_HERE"
@@ -26,5 +31,5 @@ tokenserver.fxa_browserid_issuer = "https://api-accounts.stage.mozaws.net"
 tokenserver.fxa_browserid_server_url = "https://verifier.stage.mozaws.net/v2"
 
 # cors settings
-# cors_allowed_origin = "localhost"
+cors_allowed_origin = "localhost"
 # cors_max_age = 86400
diff --git a/syncserver-db-common/Cargo.toml b/syncserver-db-common/Cargo.toml
@@ -12,6 +12,6 @@ http.workspace=true
 thiserror.workspace=true
 
 deadpool = { git = "https://github.com/mozilla-services/deadpool", tag = "deadpool-v0.7.0" }
-diesel = { version = "1.4", features = ["mysql", "r2d2"] }
-diesel_migrations = { version = "1.4.0", features = ["mysql"] }
+diesel = { version = "1.4", features = ["mysql", "sqlite","r2d2"] }
+diesel_migrations = { version = "1.4.0", features = ["mysql", "sqlite"] }
 syncserver-common = { path = "../syncserver-common" }
diff --git a/syncserver-db-common/src/error.rs b/syncserver-db-common/src/error.rs
@@ -8,14 +8,14 @@ use thiserror::Error;
 /// Error specific to any MySQL database backend. These errors are not related to the syncstorage
 /// or tokenserver application logic; rather, they are lower-level errors arising from diesel.
 #[derive(Debug)]
-pub struct MysqlError {
-    kind: MysqlErrorKind,
+pub struct SqlError {
+    kind: SqlErrorKind,
     pub status: StatusCode,
     pub backtrace: Backtrace,
 }
 
 #[derive(Debug, Error)]
-enum MysqlErrorKind {
+enum SqlErrorKind {
     #[error("A database error occurred: {}", _0)]
     DieselQuery(#[from] diesel::result::Error),
 
@@ -29,8 +29,8 @@ enum MysqlErrorKind {
     Migration(diesel_migrations::RunMigrationsError),
 }
 
-impl From<MysqlErrorKind> for MysqlError {
-    fn from(kind: MysqlErrorKind) -> Self {
+impl From<SqlErrorKind> for SqlError {
+    fn from(kind: SqlErrorKind) -> Self {
         Self {
             kind,
             status: StatusCode::INTERNAL_SERVER_ERROR,
@@ -39,21 +39,17 @@ impl From<MysqlErrorKind> for MysqlError {
     }
 }
 
-impl_fmt_display!(MysqlError, MysqlErrorKind);
+impl_fmt_display!(SqlError, SqlErrorKind);
 
-from_error!(
-    diesel::result::Error,
-    MysqlError,
-    MysqlErrorKind::DieselQuery
-);
+from_error!(diesel::result::Error, SqlError, SqlErrorKind::DieselQuery);
 from_error!(
     diesel::result::ConnectionError,
-    MysqlError,
-    MysqlErrorKind::DieselConnection
+    SqlError,
+    SqlErrorKind::DieselConnection
 );
-from_error!(diesel::r2d2::PoolError, MysqlError, MysqlErrorKind::Pool);
+from_error!(diesel::r2d2::PoolError, SqlError, SqlErrorKind::Pool);
 from_error!(
     diesel_migrations::RunMigrationsError,
-    MysqlError,
-    MysqlErrorKind::Migration
+    SqlError,
+    SqlErrorKind::Migration
 );
diff --git a/syncserver-db-common/src/test.rs b/syncserver-db-common/src/test.rs
@@ -1,6 +1,7 @@
 use diesel::{
     mysql::MysqlConnection,
     r2d2::{CustomizeConnection, Error as PoolError},
+    sqlite::SqliteConnection,
     Connection,
 };
 
@@ -12,3 +13,9 @@ impl CustomizeConnection<MysqlConnection, PoolError> for TestTransactionCustomiz
         conn.begin_test_transaction().map_err(PoolError::QueryError)
     }
 }
+
+impl CustomizeConnection<SqliteConnection, PoolError> for TestTransactionCustomizer {
+    fn on_acquire(&self, conn: &mut SqliteConnection) -> Result<(), PoolError> {
+        conn.begin_test_transaction().map_err(PoolError::QueryError)
+    }
+}
diff --git a/syncserver/Cargo.toml b/syncserver/Cargo.toml
@@ -65,3 +65,4 @@ no_auth = []
 py_verifier = ["tokenserver-auth/py"]
 mysql = ["syncstorage-db/mysql"]
 spanner = ["syncstorage-db/spanner"]
+sqlite = ["syncstorage-db/sqlite"]
diff --git a/syncserver/src/db/mod.rs b/syncserver/src/db/mod.rs
@@ -3,6 +3,7 @@
 pub mod mock;
 pub mod mysql;
 pub mod spanner;
+pub mod sqlite;
 #[cfg(test)]
 mod tests;
 pub mod transaction;
@@ -35,6 +36,11 @@ pub async fn pool_from_settings(
         "spanner" => Box::new(
             spanner::pool::SpannerDbPool::new(settings, metrics, blocking_threadpool).await?,
         ),
+        "sqlite" => Box::new(sqlite::pool::SqliteDbPool::new(
+            settings,
+            metrics,
+            blocking_threadpool,
+        )?),
         _ => Err(DbErrorKind::InvalidUrl(settings.database_url.to_owned()))?,
     })
 }
diff --git a/syncstorage-db/Cargo.toml b/syncstorage-db/Cargo.toml
@@ -24,10 +24,12 @@ syncserver-db-common = { path = "../syncserver-db-common" }
 syncserver-settings = { path = "../syncserver-settings" }
 syncstorage-db-common = { path = "../syncstorage-db-common" }
 syncstorage-mysql = { path = "../syncstorage-mysql", optional = true }
-syncstorage-settings = { path = "../syncstorage-settings" }
+syncstorage-sqlite = { path = "../syncstorage-sqlite/", optional = true}
 syncstorage-spanner = { path = "../syncstorage-spanner", optional = true }
+syncstorage-settings = { path = "../syncstorage-settings" }
 tokio = { workspace = true, features = ["macros", "sync"] }
 
 [features]
 mysql = ['syncstorage-mysql']
 spanner = ['syncstorage-spanner']
+sqlite = ['syncstorage-sqlite']
diff --git a/syncstorage-db/src/lib.rs b/syncstorage-db/src/lib.rs
@@ -15,6 +15,13 @@ pub use syncstorage_mysql::DbError;
 #[cfg(feature = "mysql")]
 pub type DbImpl = syncstorage_mysql::MysqlDb;
 
+#[cfg(feature = "sqlite")]
+pub type DbPoolImpl = syncstorage_sqlite::SqliteDbPool;
+#[cfg(feature = "sqlite")]
+pub use syncstorage_sqlite::DbError;
+#[cfg(feature = "sqlite")]
+pub type DbImpl = syncstorage_sqlite::SqliteDb;
+
 #[cfg(feature = "spanner")]
 pub type DbPoolImpl = syncstorage_spanner::SpannerDbPool;
 #[cfg(feature = "spanner")]
@@ -31,8 +38,8 @@ pub use syncstorage_db_common::{
     Db, DbPool, Sorting, UserIdentifier,
 };
 
-#[cfg(all(feature = "mysql", feature = "spanner"))]
+#[cfg(all(feature = "mysql", feature = "spanner", feature = "sqlite"))]
 compile_error!("only one of the \"mysql\" and \"spanner\" features can be enabled at a time");
 
-#[cfg(not(any(feature = "mysql", feature = "spanner")))]
+#[cfg(not(any(feature = "mysql", feature = "spanner", feature = "sqlite")))]
 compile_error!("exactly one of the \"mysql\" and \"spanner\" features must be enabled");
diff --git a/syncstorage-mysql/src/error.rs b/syncstorage-mysql/src/error.rs
@@ -3,7 +3,7 @@ use std::fmt;
 use backtrace::Backtrace;
 use http::StatusCode;
 use syncserver_common::{from_error, impl_fmt_display, InternalError, ReportableError};
-use syncserver_db_common::error::MysqlError;
+use syncserver_db_common::error::SqlError;
 use syncstorage_db_common::error::{DbErrorIntrospect, SyncstorageDbError};
 use thiserror::Error;
 
@@ -49,7 +49,7 @@ enum DbErrorKind {
     Common(SyncstorageDbError),
 
     #[error("{}", _0)]
-    Mysql(MysqlError),
+    Mysql(SqlError),
 }
 
 impl From<DbErrorKind> for DbError {
@@ -124,24 +124,24 @@ from_error!(SyncstorageDbError, DbError, DbErrorKind::Common);
 from_error!(
     diesel::result::Error,
     DbError,
-    |error: diesel::result::Error| DbError::from(DbErrorKind::Mysql(MysqlError::from(error)))
+    |error: diesel::result::Error| DbError::from(DbErrorKind::Mysql(SqlError::from(error)))
 );
 from_error!(
     diesel::result::ConnectionError,
     DbError,
-    |error: diesel::result::ConnectionError| DbError::from(DbErrorKind::Mysql(MysqlError::from(
+    |error: diesel::result::ConnectionError| DbError::from(DbErrorKind::Mysql(SqlError::from(
         error
     )))
 );
 from_error!(
     diesel::r2d2::PoolError,
     DbError,
-    |error: diesel::r2d2::PoolError| DbError::from(DbErrorKind::Mysql(MysqlError::from(error)))
+    |error: diesel::r2d2::PoolError| DbError::from(DbErrorKind::Mysql(SqlError::from(error)))
 );
 from_error!(
     diesel_migrations::RunMigrationsError,
     DbError,
     |error: diesel_migrations::RunMigrationsError| DbError::from(DbErrorKind::Mysql(
-        MysqlError::from(error)
+        SqlError::from(error)
     ))
 );
diff --git a/syncstorage-sqlite/Cargo.toml b/syncstorage-sqlite/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "syncstorage-sqlite"
+version.workspace=true
+license.workspace=true
+authors.workspace=true
+edition.workspace=true
+
+[dependencies]
+backtrace.workspace=true
+base64.workspace=true
+futures.workspace=true
+http.workspace=true
+slog-scope.workspace=true
+
+async-trait = "0.1.40"
+diesel = { version = "1.4", features = ["sqlite", "r2d2"] }
+diesel_logger = "0.1.1"
+diesel_migrations = { version = "1.4.0", features = ["sqlite"] }
+syncserver-common = { path = "../syncserver-common" }
+syncserver-db-common = { path = "../syncserver-db-common" }
+syncstorage-db-common = { path = "../syncstorage-db-common" }
+syncstorage-settings = { path = "../syncstorage-settings" }
+thiserror = "1.0.26"
+url = "2.1"
+
+[dev-dependencies]
+env_logger.workspace=true
+syncserver-settings = { path = "../syncserver-settings" }
diff --git a/syncstorage-sqlite/migrations/2024-01-19-131212_Init/down.sql b/syncstorage-sqlite/migrations/2024-01-19-131212_Init/down.sql
@@ -0,0 +1,8 @@
+-- DROP INDEX IF EXISTS `bso_expiry_idx`;
+-- DROP INDEX IF EXISTS `bso_usr_col_mod_idx`;
+
+-- DROP TABLE IF EXISTS `bso`;
+-- DROP TABLE IF EXISTS `collections`;
+-- DROP TABLE IF EXISTS `user_collections`;
+-- DROP TABLE IF EXISTS `batch_uploads`;
+-- DROP TABLE IF EXISTS `batch_upload_items`;