fix migration headaches

xmtp_mls/diesel.toml

@@ -2,7 +2,7 @@
 # see https://diesel.rs/guides/configuring-diesel-cli
 
 [print_schema]
-file = "src/storage/encrypted_store/schema.rs"
+file = "src/storage/encrypted_store/schema_gen.rs"
 
 [migrations_directory]
 dir = "migrations"

xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/down.sql

@@ -1 +1 @@
-DROP VIEW IF EXISTS conversation_list;
+DROP VIEW IF EXISTS conversation_list;

xmtp_mls/migrations/2024-12-20-143210_create_conversation_list_view/up.sql

@@ -42,6 +42,6 @@ SELECT
     rm.authority_id
 FROM
     groups g
-        LEFT JOIN ranked_messages rm
-                  ON g.id = rm.group_id AND rm.row_num = 1
-ORDER BY COALESCE(rm.sent_at_ns, g.created_at_ns) DESC;
+    LEFT JOIN ranked_messages rm
+    ON g.id = rm.group_id AND rm.row_num = 1
+ORDER BY COALESCE(rm.sent_at_ns, g.created_at_ns) DESC;

xmtp_mls/migrations/2024-12-20-214747_add_dm_id/up.sql

@@ -0,0 +1,76 @@
+DROP VIEW IF EXISTS conversation_list;
+ALTER TABLE groups ADD COLUMN dm_id TEXT;
+ALTER TABLE groups ADD COLUMN last_message_ns BIGINT;
+
+-- Fill the dm_id column
+UPDATE groups
+SET dm_id = 'dm:' ||
+    LOWER(
+        CASE
+            WHEN LOWER((SELECT inbox_id FROM identity)) < LOWER(dm_inbox_id)
+            THEN (SELECT inbox_id FROM identity) || ':' || dm_inbox_id
+            ELSE dm_inbox_id || ':' || (SELECT inbox_id FROM identity)
+        END
+    )
+WHERE dm_inbox_id IS NOT NULL;
+
+DROP INDEX IF EXISTS idx_dm_target;
+ALTER TABLE groups DROP COLUMN dm_inbox_id;
+
+-- Create a trigger to auto-update group table on insert
+CREATE TRIGGER msg_inserted
+AFTER INSERT ON group_messages
+BEGIN
+  UPDATE groups
+  SET last_message_ns = (strftime('%s', 'now') * 1000000000) + (strftime('%f', 'now') * 1000000)
+  WHERE id = NEW.group_id;
+END;
+
+
+CREATE VIEW conversation_list AS
+WITH ranked_messages AS (
+    SELECT
+        gm.group_id,
+        gm.id AS message_id,
+        gm.decrypted_message_bytes,
+        gm.sent_at_ns,
+        gm.kind AS message_kind,
+        gm.sender_installation_id,
+        gm.sender_inbox_id,
+        gm.delivery_status,
+        gm.content_type,
+        gm.version_major,
+        gm.version_minor,
+        gm.authority_id,
+        ROW_NUMBER() OVER (PARTITION BY gm.group_id ORDER BY gm.sent_at_ns DESC) AS row_num
+    FROM
+        group_messages gm
+    WHERE
+        gm.kind = 1
+)
+SELECT
+    g.id AS id,
+    g.created_at_ns,
+    g.membership_state,
+    g.installations_last_checked,
+    g.added_by_inbox_id,
+    g.welcome_id,
+    g.dm_id,
+    g.rotated_at_ns,
+    g.conversation_type,
+    rm.message_id,
+    rm.decrypted_message_bytes,
+    rm.sent_at_ns,
+    rm.message_kind,
+    rm.sender_installation_id,
+    rm.sender_inbox_id,
+    rm.delivery_status,
+    rm.content_type,
+    rm.version_major,
+    rm.version_minor,
+    rm.authority_id
+FROM
+    groups g
+    LEFT JOIN ranked_messages rm
+    ON g.id = rm.group_id AND rm.row_num = 1
+ORDER BY COALESCE(rm.sent_at_ns, g.created_at_ns) DESC;

xmtp_mls/src/storage/encrypted_store/conversation_list.rs

@@ -1,6 +1,6 @@
+use super::schema::conversation_list::dsl::conversation_list;
 use crate::storage::group::{ConversationType, GroupMembershipState};
 use crate::storage::group_message::{ContentType, DeliveryStatus, GroupMessageKind};
-use crate::storage::schema::conversation_list::dsl::conversation_list;
 use crate::storage::{DbConnection, StorageError};
 use diesel::{QueryDsl, Queryable, RunQueryDsl, Table};
 use serde::{Deserialize, Serialize};
@@ -23,7 +23,7 @@ pub struct ConversationListItem {
     /// The sequence id of the welcome message
     pub welcome_id: Option<i64>,
     /// The inbox_id of the DM target
-    pub dm_inbox_id: Option<String>,
+    pub dm_id: Option<String>,
     /// The last time the leaf node encryption key was rotated
     pub rotated_at_ns: i64,
     /// Enum, [`ConversationType`] signifies the group conversation type which extends to who can access it.

xmtp_mls/src/storage/encrypted_store/mod.rs

@@ -25,6 +25,7 @@ pub mod key_store_entry;
 mod native;
 pub mod refresh_state;
 pub mod schema;
+mod schema_gen;
 #[cfg(not(target_arch = "wasm32"))]
 mod sqlcipher_connection;
 pub mod user_preferences;

xmtp_mls/src/storage/encrypted_store/schema.rs

@@ -1,133 +1,7 @@
-// @generated automatically by Diesel CLI.
+pub use super::schema_gen::*;
 
 diesel::table! {
-    association_state (inbox_id, sequence_id) {
-        inbox_id -> Text,
-        sequence_id -> BigInt,
-        state -> Binary,
-    }
-}
-
-diesel::table! {
-    consent_records (entity_type, entity) {
-        entity_type -> Integer,
-        state -> Integer,
-        entity -> Text,
-    }
-}
-
-diesel::table! {
-    group_intents (id) {
-        id -> Integer,
-        kind -> Integer,
-        group_id -> Binary,
-        data -> Binary,
-        state -> Integer,
-        payload_hash -> Nullable<Binary>,
-        post_commit_data -> Nullable<Binary>,
-        publish_attempts -> Integer,
-        staged_commit -> Nullable<Binary>,
-        published_in_epoch -> Nullable<BigInt>,
-    }
-}
-
-diesel::table! {
-    group_messages (id) {
-        id -> Binary,
-        group_id -> Binary,
-        decrypted_message_bytes -> Binary,
-        sent_at_ns -> BigInt,
-        kind -> Integer,
-        sender_installation_id -> Binary,
-        sender_inbox_id -> Text,
-        delivery_status -> Integer,
-        content_type -> Integer,
-        version_minor -> Integer,
-        version_major -> Integer,
-        authority_id -> Text,
-    }
-}
-
-diesel::table! {
-    groups (id) {
-        id -> Binary,
-        created_at_ns -> BigInt,
-        membership_state -> Integer,
-        installations_last_checked -> BigInt,
-        added_by_inbox_id -> Text,
-        welcome_id -> Nullable<BigInt>,
-        rotated_at_ns -> BigInt,
-        conversation_type -> Integer,
-        dm_id -> Nullable<Text>,
-        last_message_ns -> Nullable<BigInt>,
-    }
-}
-
-diesel::table! {
-    identity (rowid) {
-        inbox_id -> Text,
-        installation_keys -> Binary,
-        credential_bytes -> Binary,
-        rowid -> Nullable<Integer>,
-    }
-}
-
-diesel::table! {
-    identity_updates (inbox_id, sequence_id) {
-        inbox_id -> Text,
-        sequence_id -> BigInt,
-        server_timestamp_ns -> BigInt,
-        payload -> Binary,
-    }
-}
-
-diesel::table! {
-    key_package_history (id) {
-        id -> Integer,
-        key_package_hash_ref -> Binary,
-        created_at_ns -> BigInt,
-    }
-}
-
-diesel::table! {
-    openmls_key_store (key_bytes) {
-        key_bytes -> Binary,
-        value_bytes -> Binary,
-    }
-}
-
-diesel::table! {
-    openmls_key_value (version, key_bytes) {
-        version -> Integer,
-        key_bytes -> Binary,
-        value_bytes -> Binary,
-    }
-}
-
-diesel::table! {
-    refresh_state (entity_id, entity_kind) {
-        entity_id -> Binary,
-        entity_kind -> Integer,
-        cursor -> BigInt,
-    }
-}
-
-diesel::table! {
-    user_preferences (id) {
-        id -> Integer,
-        hmac_key -> Nullable<Binary>,
-    }
-}
-
-diesel::table! {
-    wallet_addresses (wallet_address) {
-        inbox_id -> Text,
-        wallet_address -> Text,
-    }
-}
-
-diesel::table! {
-    conversation_list (id) {
+  conversation_list (id) {
     id -> Binary,
     created_at_ns -> BigInt,
     membership_state -> Integer,
@@ -148,25 +22,5 @@ diesel::table! {
     version_major -> Nullable<Integer>,
     version_minor -> Nullable<Integer>,
     authority_id -> Nullable<Text>
-    }
+  }
 }
-
-diesel::joinable!(group_intents -> groups (group_id));
-diesel::joinable!(group_messages -> groups (group_id));
-
-diesel::allow_tables_to_appear_in_same_query!(
-    association_state,
-    consent_records,
-    group_intents,
-    group_messages,
-    groups,
-    identity,
-    identity_updates,
-    key_package_history,
-    openmls_key_store,
-    openmls_key_value,
-    refresh_state,
-    user_preferences,
-    wallet_addresses,
-    conversation_list
-);