fix: Pass some SQLite tests for syncstorage

Makefile

@@ -82,13 +82,13 @@ run_spanner: python
 		cargo run --no-default-features --features=spanner,py_verifier -- --config config/local.toml
 
 test_mysql:
-	SYNC_SYNCSTORAGE__DATABASE_URL=mysql://sample_user:sample_password@localhost/syncstorage_rs \
-		SYNC_TOKENSERVER__DATABASE_URL=mysql://sample_user:sample_password@localhost/tokenserver_rs \
+	SYNC_SYNCSTORAGE__DATABASE_URL=mysql://eragon:@localhost/syncstorage \
+		SYNC_TOKENSERVER__DATABASE_URL=mysql://eragon:@localhost/syncstorage \
 		RUST_TEST_THREADS=1 \
 		cargo test --workspace --no-default-features --features=mysql,py_verifier
 
 test_sqlite:
-	SYNC_SYNCSTORAGE__DATABASE_URL=sqlite://:memory: \
-		SYNC_TOKENSERVER__DATABASE_URL=sqlite://:memory: \
+	SYNC_SYNCSTORAGE__DATABASE_URL=sqlite:///tmp/syncstorage.db\
+		SYNC_TOKENSERVER__DATABASE_URL=sqlite:///tmp/tokenserver.db \
 		RUST_TEST_THREADS=1 \
 		cargo test --workspace --no-default-features --features=sqlite,py_verifier

syncstorage-sqlite/src/batch.rs

@@ -147,7 +147,6 @@ pub fn commit(db: &SqliteDb, params: params::CommitBatch) -> DbResult<results::C
         .bind::<BigInt, _>(&batch_id)
         .bind::<BigInt, _>(user_id)
         .bind::<BigInt, _>(&db.timestamp().as_i64())
-        .bind::<BigInt, _>(&db.timestamp().as_i64())
         .execute(&db.conn)?;
 
     db.update_collection(user_id as u32, collection_id)?;
@@ -181,7 +180,7 @@ pub fn do_append(
     }
 
     // It's possible for the list of items to contain a duplicate key entry.
-    // This means that we can't really call `ON DUPLICATE` here, because that's
+    // This means that we can't really call `ON CONFLICT` here, because that's
     // more about inserting one item at a time. (e.g. it works great if the
     // values contain a key that's already in the database, less so if the
     // the duplicate is in the value set we're inserting.
@@ -242,7 +241,6 @@ pub fn do_append(
                     batch_upload_items::sortindex.eq(bso.sortindex),
                     batch_upload_items::payload.eq(bso.payload),
                     batch_upload_items::payload_size.eq(payload_size),
-                    batch_upload_items::ttl_offset.eq(bso.ttl.map(|ttl| ttl as i32)),
                 ))
                 .execute(&db.conn)?;
             // make sure to include the key into our table check.

syncstorage-sqlite/src/batch_commit.sql

@@ -5,15 +5,15 @@ SELECT
        id,
        ?,
        sortindex,
-       COALESCE((ttl_offset * 1000) + ?, ?),
-       COALESCE(payload, ''),
-       COALESCE(payload_size, 0)
+       COALESCE((ttl_offset * 1000) + ?, ?) as ttl,
+       COALESCE(payload, '') as payload,
+       COALESCE(payload_size, 0) as payload_size
   FROM batch_upload_items
  WHERE batch = ?
    AND userid = ?
-    ON DUPLICATE KEY UPDATE
+    ON CONFLICT(userid, collection, id) DO UPDATE SET
        modified = ?,
-       sortindex = COALESCE(batch_upload_items.sortindex, bso.sortindex),
-       ttl = COALESCE((batch_upload_items.ttl_offset * 1000) + ?, bso.ttl),
-       payload = COALESCE(batch_upload_items.payload, bso.payload),
-       payload_size = COALESCE(batch_upload_items.payload_size, bso.payload_size)
+       sortindex = COALESCE(excluded.sortindex, bso.sortindex),
+       ttl = COALESCE(excluded.ttl, bso.ttl),
+       payload = COALESCE(NULLIF(excluded.payload, ''), bso.payload),
+       payload_size = COALESCE(excluded.payload_size, bso.payload_size)

syncstorage-sqlite/src/diesel_ext.rs

@@ -1,3 +1,5 @@
+use core::fmt;
+
 use diesel::{
     backend::Backend,
     insertable::CanInsertInSingleQuery,
@@ -28,12 +30,13 @@ where
     U: QueryFragment<DB> + CanInsertInSingleQuery<DB>,
     Op: QueryFragment<DB>,
     Ret: QueryFragment<DB>,
-    X: Expression,
+    X: Expression + fmt::Debug,
 {
     fn walk_ast(&self, mut out: AstPass<'_, DB>) -> QueryResult<()> {
         self.0.walk_ast(out.reborrow())?;
-        out.push_sql(" ON DUPLICATE KEY UPDATE ");
+        out.push_sql(" ON CONFLICT({user_id}, {collection_id}) DO UPDATE SET ");
         //self.1.walk_ast(out.reborrow())?;
+        debug!("{:?}", self.1);
         Ok(())
     }
 }

tokenserver-db-mysql/src/lib.rs

@@ -2,5 +2,4 @@ extern crate diesel;
 #[macro_use]
 extern crate diesel_migrations;
 
-//pub mod models;
 pub mod pool;