Identity DB Schema (#739)

* New DB schema

* Rename identity table

bindings_ffi/src/mls.rs

@@ -589,7 +589,7 @@ pub struct FfiMessage {
     pub id: Vec<u8>,
     pub sent_at_ns: i64,
     pub convo_id: Vec<u8>,
-    pub addr_from: String,
+    pub sender_inbox_id: String,
     pub content: Vec<u8>,
     pub kind: FfiGroupMessageKind,
     pub delivery_status: FfiDeliveryStatus,
@@ -601,7 +601,7 @@ impl From<StoredGroupMessage> for FfiMessage {
             id: msg.id,
             sent_at_ns: msg.sent_at_ns,
             convo_id: msg.group_id,
-            addr_from: msg.sender_account_address,
+            sender_inbox_id: msg.sender_inbox_id,
             content: msg.decrypted_message_bytes,
             kind: msg.kind.into(),
             delivery_status: msg.delivery_status.into(),

examples/cli/cli-client.rs

@@ -417,10 +417,11 @@ fn format_messages(
         if text.is_none() {
             continue;
         }
-        let sender = if msg.sender_account_address == my_account_address {
+        // TODO:nm use inbox ID
+        let sender = if msg.sender_inbox_id == my_account_address {
             "Me".to_string()
         } else {
-            msg.sender_account_address
+            msg.sender_inbox_id
         };
 
         let msg_line = format!(

examples/cli/serializable.rs

@@ -49,7 +49,7 @@ impl<'a> From<&'a MlsGroup> for SerializableGroup {
 
 #[derive(Serialize, Debug, Clone)]
 pub struct SerializableMessage {
-    sender_account_address: String,
+    sender_inbox_id: String,
     sent_at_ns: u64,
     message_text: Option<String>,
     // content_type: String
@@ -59,7 +59,7 @@ impl SerializableMessage {
     pub fn from_stored_message(msg: &StoredGroupMessage) -> Self {
         let maybe_text = maybe_get_text(msg);
         Self {
-            sender_account_address: msg.sender_account_address.clone(),
+            sender_inbox_id: msg.sender_inbox_id.clone(),
             sent_at_ns: msg.sent_at_ns as u64,
             message_text: maybe_text,
         }

xmtp_mls/migrations/2024-05-15-145138_new_schema/down.sql

@@ -0,0 +1,15 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE IF EXISTS "groups";
+
+DROP TABLE IF EXISTS group_messages;
+
+DROP TABLE IF EXISTS refresh_state;
+
+DROP TABLE IF EXISTS group_intents;
+
+DROP TABLE IF EXISTS identity_updates;
+
+DROP TABLE IF EXISTS identity_inbox;
+
+DROP TABLE IF EXISTS openmls_key_store;
+

xmtp_mls/migrations/2024-05-15-145138_new_schema/up.sql

@@ -0,0 +1,104 @@
+-- Your SQL goes here
+CREATE TABLE openmls_key_store(
+    key_bytes BLOB PRIMARY KEY NOT NULL,
+    value_bytes BLOB NOT NULL
+);
+
+CREATE TABLE "identity"(
+    "inbox_id" text NOT NULL,
+    "installation_keys" BLOB NOT NULL,
+    "credential_bytes" BLOB NOT NULL,
+    rowid integer PRIMARY KEY CHECK (rowid = 1) -- There can only be one identity
+);
+
+CREATE TABLE "groups"(
+    -- Random ID generated by group creator
+    "id" BLOB PRIMARY KEY NOT NULL,
+    -- Based on the timestamp of the welcome message
+    "created_at_ns" bigint NOT NULL,
+    -- Enum of GROUP_MEMBERSHIP_STATE
+    "membership_state" int NOT NULL,
+    -- Last time the installations were checked for the purpose of seeing if any are missing
+    "installations_last_checked" bigint NOT NULL,
+    -- Values are 1 = Conversation, 2 = Sync
+    "purpose" int NOT NULL DEFAULT 1,
+    -- Which inbox added you to the group
+    "added_by_inbox_id" text NOT NULL
+);
+
+-- Allow for efficient sorting of groups
+CREATE INDEX groups_created_at_idx ON GROUPS (created_at_ns);
+
+-- This index allows you to filter by membership_state and then created_at_ns
+CREATE INDEX groups_membership_state_created_at_idx ON GROUPS (membership_state, created_at_ns);
+
+-- Successfully processed messages meant to be returned to the user
+CREATE TABLE group_messages(
+    -- Derived via generate_message_id() in SDK, which hashes several inputs
+    "id" BLOB PRIMARY KEY NOT NULL,
+    "group_id" BLOB NOT NULL,
+    -- Message contents after decryption
+    "decrypted_message_bytes" BLOB NOT NULL,
+    -- Based on the timestamp of the message
+    "sent_at_ns" bigint NOT NULL,
+    -- Enum GROUP_MESSAGE_KIND
+    "kind" int NOT NULL,
+    -- Could remove this if we added a table mapping installation_ids to wallet addresses
+    "sender_installation_id" BLOB NOT NULL,
+    -- The inbox_id of the sender
+    "sender_inbox_id" text NOT NULL,
+    -- Values are: 1 = Published, 2 = Unpublished
+    "delivery_status" int NOT NULL DEFAULT 1,
+    FOREIGN KEY (group_id) REFERENCES "groups"(id)
+);
+
+CREATE INDEX group_messages_group_id_sort_idx ON group_messages(group_id, sent_at_ns);
+
+-- Used to keep track of the last seen message timestamp in a topic
+CREATE TABLE refresh_state(
+    -- E.g. the Id of the group
+    "entity_id" BLOB NOT NULL,
+    -- Welcomes or other types
+    "entity_kind" integer NOT NULL, -- Need to allow for groups and welcomes to be separated, since a malicious client could manipulate their group ID to match someone's installation_id and make a mess
+    -- Where you are in the topic
+    "cursor" bigint NOT NULL,
+    PRIMARY KEY (entity_id, entity_kind)
+);
+
+-- This table is required to retry messages that do not send successfully due to epoch conflicts
+CREATE TABLE group_intents(
+    -- Serial ID auto-generated by the DB
+    "id" integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+    -- Enum INTENT_KIND
+    "kind" int NOT NULL,
+    "group_id" BLOB NOT NULL,
+    -- Some sort of serializable blob that can be used to re-try the message if the first attempt failed due to conflict
+    "data" BLOB NOT NULL,
+    -- INTENT_STATE,
+    "state" int NOT NULL,
+    -- The hash of the encrypted, concrete, form of the message if it was published.
+    "payload_hash" BLOB UNIQUE,
+    -- (Optional) data needed for the post-commit flow. For example, welcome messages
+    "post_commit_data" BLOB,
+    -- The number of publish attempts
+    "publish_attempts" int NOT NULL DEFAULT 0,
+    FOREIGN KEY (group_id) REFERENCES "groups"(id)
+);
+
+CREATE INDEX group_intents_group_id_state ON group_intents(group_id, state);
+
+CREATE TABLE identity_updates(
+    -- The inbox_id the update refers to
+    "inbox_id" text NOT NULL,
+    -- The sequence_id of the update
+    "sequence_id" bigint NOT NULL,
+    -- Based on the timestamp given by the server
+    "server_timestamp_ns" bigint NOT NULL,
+    -- Random ID generated by group creator
+    "payload" BLOB NOT NULL,
+    -- Compound primary key of the `inbox_id` and `sequence_id`
+    PRIMARY KEY (inbox_id, sequence_id)
+);
+
+CREATE INDEX idx_identity_updates_inbox_id_sequence_id_asc ON identity_updates(inbox_id, sequence_id ASC);
+

xmtp_mls/src/groups/mod.rs

@@ -407,8 +407,7 @@ impl MlsGroup {
             sent_at_ns: now,
             kind: GroupMessageKind::Application,
             sender_installation_id: self.context.installation_public_key(),
-            // TODO: Remove this hack
-            sender_account_address: self.context.inbox_id(),
+            sender_inbox_id: self.context.inbox_id(),
             delivery_status: DeliveryStatus::Unpublished,
         };
         group_message.store(&conn)?;
@@ -566,7 +565,7 @@ impl MlsGroup {
             .map_err(GroupError::from)
             .and_then(|fetch_result| {
                 fetch_result
-                    .map(|group| group.added_by_address.clone())
+                    .map(|group| group.added_by_inbox_id.clone())
                     .ok_or_else(|| GroupError::GroupNotFound)
             })
     }