feat: add new sqlite backend based on mysql code

- Remove diesel_ext, which only contained barely used extensions for mysql
- Merge migrations for sqlite
- Alter queries that differ in sqlite (mainly UPSERT statements)

Cargo.toml

@@ -28,7 +28,7 @@ cadence = "0.22"
 chrono = "0.4"
 config = "0.10"
 deadpool = "0.6"
-diesel = { version = "1.4", features = ["mysql", "r2d2"] }
+diesel = { version = "1.4", features = ["mysql", "r2d2", "sqlite"] }
 diesel_logger = "0.1.1"
 diesel_migrations = { version = "1.4.0", features = ["mysql"] }
 docopt = "1.1.0"

README.md

@@ -12,6 +12,7 @@ Mozilla Sync Storage built with [Rust](https://rust-lang.org).
 - [Local Setup](#local-setup)
   - [MySQL](#mysql)
   - [Spanner](#spanner)
+  - [Sqlite](#sqlite)
   - [Running via Docker](#running-via-docker)
   - [Connecting to Firefox](#connecting-to-firefox)
 - [Logging](#logging)
@@ -106,6 +107,15 @@ To point to a GCP hosted Spanner instance from your local machine, follow these
 4. `make run_spanner`.
 5. Visit `http://localhost:8000/__heartbeat__` to make sure the server is running.
 
+### Sqlite
+
+Setting up the server with sqlite only requires a path to the database file, 
+which will be created automatically:
+
+`sqlite:path/syncdb.sqlite`
+
+This requires at least sqlite v3.24.0 to be installed on the host system.
+
 ### Running via Docker
 This requires access to the mozilla-rust-sdk which is now available at `/vendor/mozilla-rust-adk`.
 

migrations/sqlite/2020-11-29-124806_init/down.sql

@@ -0,0 +1,8 @@
+-- DROP INDEX IF EXISTS `bso_expiry_idx`;
+-- DROP INDEX IF EXISTS `bso_usr_col_mod_idx`;
+
+-- DROP TABLE IF EXISTS `bso`;
+-- DROP TABLE IF EXISTS `collections`;
+-- DROP TABLE IF EXISTS `user_collections`;
+-- DROP TABLE IF EXISTS `batch_uploads`;
+-- DROP TABLE IF EXISTS `batch_upload_items`;

migrations/sqlite/2020-11-29-124806_init/up.sql

@@ -0,0 +1,79 @@
+-- XXX: bsov1, etc
+-- We use Bigint for some fields instead of Integer, even though Sqlite does not have the concept of Bigint,
+-- to allow diesel to assume that integer can be mapped to i64. See https://github.com/diesel-rs/diesel/issues/852
+
+
+CREATE TABLE IF NOT EXISTS `bso`
+(
+    `userid`       BIGINT  NOT NULL,
+    `collection`   INTEGER NOT NULL,
+    `id`           TEXT    NOT NULL,
+
+    `sortindex`    INTEGER,
+
+    `payload`      TEXT    NOT NULL,
+    `payload_size` BIGINT DEFAULT 0,
+
+    -- last modified time in milliseconds since epoch
+    `modified`     BIGINT  NOT NULL,
+    -- expiration in milliseconds since epoch
+    `ttl`          BIGINT DEFAULT '3153600000000' NOT NULL,
+
+    PRIMARY KEY (`userid`, `collection`, `id`)
+);
+CREATE INDEX IF NOT EXISTS `bso_expiry_idx` ON `bso` (`ttl`);
+CREATE INDEX IF NOT EXISTS `bso_usr_col_mod_idx` ON `bso` (`userid`, `collection`, `modified`);
+
+CREATE TABLE IF NOT EXISTS `collections`
+(
+    `id`   INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    `name` TEXT UNIQUE                       NOT NULL
+);
+INSERT INTO collections (id, name)
+VALUES (1, 'clients'),
+       (2, 'crypto'),
+       (3, 'forms'),
+       (4, 'history'),
+       (5, 'keys'),
+       (6, 'meta'),
+       (7, 'bookmarks'),
+       (8, 'prefs'),
+       (9, 'tabs'),
+       (10, 'passwords'),
+       (11, 'addons'),
+       (12, 'addresses'),
+       (13, 'creditcards'),
+       -- Reserve space for additions to the standard collections
+       (100, '');
+
+
+CREATE TABLE IF NOT EXISTS `user_collections`
+(
+    `userid`        BIGINT  NOT NULL,
+    `collection`    INTEGER NOT NULL,
+    -- last modified time in milliseconds since epoch
+    `last_modified` BIGINT  NOT NULL,
+    `total_bytes`   BIGINT,
+    `count`         INTEGER,
+    PRIMARY KEY (`userid`, `collection`)
+);
+
+CREATE TABLE IF NOT EXISTS `batch_uploads`
+(
+    `batch`      BIGINT  NOT NULL,
+    `userid`     BIGINT  NOT NULL,
+    `collection` INTEGER NOT NULL,
+    PRIMARY KEY (`batch`, `userid`)
+);
+
+CREATE TABLE IF NOT EXISTS `batch_upload_items`
+(
+    `batch`        BIGINT NOT NULL,
+    `userid`       BIGINT NOT NULL,
+    `id`           TEXT   NOT NULL,
+    `sortindex`    INTEGER DEFAULT NULL,
+    `payload`      TEXT,
+    `payload_size` BIGINT  DEFAULT NULL,
+    `ttl_offset`   INTEGER DEFAULT NULL,
+    PRIMARY KEY (`batch`, `userid`, `id`)
+);

src/db/mod.rs

@@ -6,6 +6,7 @@ pub mod mysql;
 pub mod params;
 pub mod results;
 pub mod spanner;
+pub mod sqlite;
 #[cfg(test)]
 mod tests;
 pub mod transaction;
@@ -273,6 +274,7 @@ pub async fn pool_from_settings(
     Ok(match url.scheme() {
         "mysql" => Box::new(mysql::pool::MysqlDbPool::new(&settings, &metrics)?),
         "spanner" => Box::new(spanner::pool::SpannerDbPool::new(&settings, &metrics).await?),
+        "sqlite" => Box::new(sqlite::pool::SqliteDbPool::new(&settings, &metrics)?),
         _ => Err(DbErrorKind::InvalidUrl(settings.database_url.to_owned()))?,
     })
 }