From e2c125226b4b20c314c5312acf014b9e256856f2 Mon Sep 17 00:00:00 2001 From: Nicholas Molnar <65710+neekolas@users.noreply.github.com> Date: Wed, 15 May 2024 15:15:33 -0700 Subject: [PATCH] Identity DB Schema (#739) * New DB schema * Rename identity table --- bindings_ffi/src/mls.rs | 4 +- examples/cli/cli-client.rs | 5 +- examples/cli/serializable.rs | 4 +- .../down.sql | 1 - .../2023-10-24-213844_create_key_store/up.sql | 4 - .../down.sql | 1 - .../2023-10-25-234319_create_identity/up.sql | 6 - .../down.sql | 9 -- .../up.sql | 69 ------------ .../down.sql | 2 - .../up.sql | 2 - .../down.sql | 9 -- .../up.sql | 3 - .../2024-03-15-152716_group_types/down.sql | 9 -- .../2024-03-15-152716_group_types/up.sql | 2 - .../down.sql | 9 -- .../up.sql | 2 - .../2024-04-11-004240_identity_init/down.sql | 1 - .../2024-04-11-004240_identity_init/up.sql | 14 --- .../2024-04-30-035609_identity_inbox/down.sql | 1 - .../2024-04-30-035609_identity_inbox/up.sql | 6 - .../2024-05-15-145138_new_schema/down.sql | 15 +++ .../2024-05-15-145138_new_schema/up.sql | 104 ++++++++++++++++++ xmtp_mls/src/groups/mod.rs | 5 +- xmtp_mls/src/groups/sync.rs | 14 ++- xmtp_mls/src/storage/encrypted_store/group.rs | 11 +- .../storage/encrypted_store/group_message.rs | 6 +- .../src/storage/encrypted_store/identity.rs | 8 +- .../src/storage/encrypted_store/schema.rs | 14 +-- xmtp_mls/update-C72KZAOXqn6bElzI.db3-shm | Bin 0 -> 32768 bytes xmtp_mls/update-C72KZAOXqn6bElzI.db3-wal | 0 31 files changed, 152 insertions(+), 188 deletions(-) delete mode 100644 xmtp_mls/migrations/2023-10-24-213844_create_key_store/down.sql delete mode 100644 xmtp_mls/migrations/2023-10-24-213844_create_key_store/up.sql delete mode 100644 xmtp_mls/migrations/2023-10-25-234319_create_identity/down.sql delete mode 100644 xmtp_mls/migrations/2023-10-25-234319_create_identity/up.sql delete mode 100644 xmtp_mls/migrations/2023-10-29-205333_state_machine_init/down.sql delete mode 100644 xmtp_mls/migrations/2023-10-29-205333_state_machine_init/up.sql delete mode 100644 xmtp_mls/migrations/2024-02-23-170839_update_installations_column/down.sql delete mode 100644 xmtp_mls/migrations/2024-02-23-170839_update_installations_column/up.sql delete mode 100644 xmtp_mls/migrations/2024-02-23-174003_add_msg_delivery_status/down.sql delete mode 100644 xmtp_mls/migrations/2024-02-23-174003_add_msg_delivery_status/up.sql delete mode 100644 xmtp_mls/migrations/2024-03-15-152716_group_types/down.sql delete mode 100644 xmtp_mls/migrations/2024-03-15-152716_group_types/up.sql delete mode 100644 xmtp_mls/migrations/2024-04-08-180113_group_added_by_address/down.sql delete mode 100644 xmtp_mls/migrations/2024-04-08-180113_group_added_by_address/up.sql delete mode 100644 xmtp_mls/migrations/2024-04-11-004240_identity_init/down.sql delete mode 100644 xmtp_mls/migrations/2024-04-11-004240_identity_init/up.sql delete mode 100644 xmtp_mls/migrations/2024-04-30-035609_identity_inbox/down.sql delete mode 100644 xmtp_mls/migrations/2024-04-30-035609_identity_inbox/up.sql create mode 100644 xmtp_mls/migrations/2024-05-15-145138_new_schema/down.sql create mode 100644 xmtp_mls/migrations/2024-05-15-145138_new_schema/up.sql create mode 100644 xmtp_mls/update-C72KZAOXqn6bElzI.db3-shm create mode 100644 xmtp_mls/update-C72KZAOXqn6bElzI.db3-wal diff --git a/bindings_ffi/src/mls.rs b/bindings_ffi/src/mls.rs index 89735772c..247c6bbfd 100644 --- a/bindings_ffi/src/mls.rs +++ b/bindings_ffi/src/mls.rs @@ -589,7 +589,7 @@ pub struct FfiMessage { pub id: Vec, pub sent_at_ns: i64, pub convo_id: Vec, - pub addr_from: String, + pub sender_inbox_id: String, pub content: Vec, pub kind: FfiGroupMessageKind, pub delivery_status: FfiDeliveryStatus, @@ -601,7 +601,7 @@ impl From for FfiMessage { id: msg.id, sent_at_ns: msg.sent_at_ns, convo_id: msg.group_id, - addr_from: msg.sender_account_address, + sender_inbox_id: msg.sender_inbox_id, content: msg.decrypted_message_bytes, kind: msg.kind.into(), delivery_status: msg.delivery_status.into(), diff --git a/examples/cli/cli-client.rs b/examples/cli/cli-client.rs index 68d7a7e94..0557c23a7 100644 --- a/examples/cli/cli-client.rs +++ b/examples/cli/cli-client.rs @@ -417,10 +417,11 @@ fn format_messages( if text.is_none() { continue; } - let sender = if msg.sender_account_address == my_account_address { + // TODO:nm use inbox ID + let sender = if msg.sender_inbox_id == my_account_address { "Me".to_string() } else { - msg.sender_account_address + msg.sender_inbox_id }; let msg_line = format!( diff --git a/examples/cli/serializable.rs b/examples/cli/serializable.rs index 409a08cff..9d6ec2efb 100644 --- a/examples/cli/serializable.rs +++ b/examples/cli/serializable.rs @@ -49,7 +49,7 @@ impl<'a> From<&'a MlsGroup> for SerializableGroup { #[derive(Serialize, Debug, Clone)] pub struct SerializableMessage { - sender_account_address: String, + sender_inbox_id: String, sent_at_ns: u64, message_text: Option, // content_type: String @@ -59,7 +59,7 @@ impl SerializableMessage { pub fn from_stored_message(msg: &StoredGroupMessage) -> Self { let maybe_text = maybe_get_text(msg); Self { - sender_account_address: msg.sender_account_address.clone(), + sender_inbox_id: msg.sender_inbox_id.clone(), sent_at_ns: msg.sent_at_ns as u64, message_text: maybe_text, } diff --git a/xmtp_mls/migrations/2023-10-24-213844_create_key_store/down.sql b/xmtp_mls/migrations/2023-10-24-213844_create_key_store/down.sql deleted file mode 100644 index d7c804b80..000000000 --- a/xmtp_mls/migrations/2023-10-24-213844_create_key_store/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE openmls_key_store; diff --git a/xmtp_mls/migrations/2023-10-24-213844_create_key_store/up.sql b/xmtp_mls/migrations/2023-10-24-213844_create_key_store/up.sql deleted file mode 100644 index 8f4b81a93..000000000 --- a/xmtp_mls/migrations/2023-10-24-213844_create_key_store/up.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE IF NOT EXISTS openmls_key_store ( - key_bytes BLOB PRIMARY KEY NOT NULL, - value_bytes BLOB NOT NULL -); diff --git a/xmtp_mls/migrations/2023-10-25-234319_create_identity/down.sql b/xmtp_mls/migrations/2023-10-25-234319_create_identity/down.sql deleted file mode 100644 index ca9f66bf9..000000000 --- a/xmtp_mls/migrations/2023-10-25-234319_create_identity/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE identity; diff --git a/xmtp_mls/migrations/2023-10-25-234319_create_identity/up.sql b/xmtp_mls/migrations/2023-10-25-234319_create_identity/up.sql deleted file mode 100644 index 9934ee26b..000000000 --- a/xmtp_mls/migrations/2023-10-25-234319_create_identity/up.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE identity ( - account_address TEXT NOT NULL, - installation_keys BLOB NOT NULL, - credential_bytes BLOB NOT NULL, - rowid INTEGER PRIMARY KEY CHECK (rowid = 1) -- There can only be one identity -); diff --git a/xmtp_mls/migrations/2023-10-29-205333_state_machine_init/down.sql b/xmtp_mls/migrations/2023-10-29-205333_state_machine_init/down.sql deleted file mode 100644 index d0cf6f5f7..000000000 --- a/xmtp_mls/migrations/2023-10-29-205333_state_machine_init/down.sql +++ /dev/null @@ -1,9 +0,0 @@ -DROP TABLE IF EXISTS groups; - -DROP TABLE IF EXISTS group_messages; - -DROP TABLE IF EXISTS topic_refresh_state; - -DROP TABLE IF EXISTS group_intents; - -DROP TABLE IF EXISTS outbound_welcome_messages; \ No newline at end of file diff --git a/xmtp_mls/migrations/2023-10-29-205333_state_machine_init/up.sql b/xmtp_mls/migrations/2023-10-29-205333_state_machine_init/up.sql deleted file mode 100644 index 96cda90fc..000000000 --- a/xmtp_mls/migrations/2023-10-29-205333_state_machine_init/up.sql +++ /dev/null @@ -1,69 +0,0 @@ -CREATE TABLE groups ( - -- Random ID generated by group creator - "id" BLOB PRIMARY KEY NOT NULL, - -- Based on the timestamp of the welcome message - "created_at_ns" BIGINT NOT NULL, - -- Enum of GROUP_MEMBERSHIP_STATE - "membership_state" INT NOT NULL, - -- Last time the installations were checked for the purpose of seeing if any are missing - "installation_list_last_checked" BIGINT NOT NULL -); - --- Allow for efficient sorting of groups -CREATE INDEX groups_created_at_idx ON groups(created_at_ns); - -CREATE INDEX groups_membership_state ON groups(membership_state); - --- Successfully processed messages meant to be returned to the user -CREATE TABLE group_messages ( - -- Derived via SHA256(CONCAT(decrypted_message_bytes, conversation_id, timestamp)) - "id" BLOB PRIMARY KEY NOT NULL, - "group_id" BLOB NOT NULL, - -- Message contents after decryption - "decrypted_message_bytes" BLOB NOT NULL, - -- Based on the timestamp of the message - "sent_at_ns" BIGINT NOT NULL, - -- Enum GROUP_MESSAGE_KIND - "kind" INT NOT NULL, - -- Could remove this if we added a table mapping installation_ids to wallet addresses - "sender_installation_id" BLOB NOT NULL, - "sender_account_address" TEXT NOT NULL, - FOREIGN KEY (group_id) REFERENCES groups(id) -); - -CREATE INDEX group_messages_group_id_sort_idx ON group_messages(group_id, sent_at_ns); - --- Used to keep track of the last seen message timestamp in a topic -CREATE TABLE refresh_state ( - -- E.g. the Id of the group - "entity_id" BLOB NOT NULL, - -- Welcomes or other types - "entity_kind" INTEGER NOT NULL, -- Need to allow for groups and welcomes to be separated, since a malicious client could manipulate their group ID to match someone's installation_id and make a mess - -- Where you are in the topic - "cursor" BIGINT NOT NULL, - - PRIMARY KEY (entity_id, entity_kind) -); - --- This table is required to retry messages that do not send successfully due to epoch conflicts -CREATE TABLE group_intents ( - -- Serial ID auto-generated by the DB - "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, - -- Enum INTENT_KIND - "kind" INT NOT NULL, - "group_id" BLOB NOT NULL, - -- Some sort of serializable blob that can be used to re-try the message if the first attempt failed due to conflict - "data" BLOB NOT NULL, - -- INTENT_STATE, - "state" INT NOT NULL, - -- The hash of the encrypted, concrete, form of the message if it was published. - "payload_hash" BLOB UNIQUE, - -- (Optional) data needed for the post-commit flow. For example, welcome messages - "post_commit_data" BLOB, - -- The number of publish attempts - "publish_attempts" INT NOT NULL DEFAULT 0, - - FOREIGN KEY (group_id) REFERENCES groups(id) -); - -CREATE INDEX group_intents_group_id_state ON group_intents(group_id, state); diff --git a/xmtp_mls/migrations/2024-02-23-170839_update_installations_column/down.sql b/xmtp_mls/migrations/2024-02-23-170839_update_installations_column/down.sql deleted file mode 100644 index ab9dc54d8..000000000 --- a/xmtp_mls/migrations/2024-02-23-170839_update_installations_column/down.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE groups -RENAME COLUMN installations_last_checked TO installation_list_last_checked; diff --git a/xmtp_mls/migrations/2024-02-23-170839_update_installations_column/up.sql b/xmtp_mls/migrations/2024-02-23-170839_update_installations_column/up.sql deleted file mode 100644 index d5291616b..000000000 --- a/xmtp_mls/migrations/2024-02-23-170839_update_installations_column/up.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE groups -RENAME COLUMN installation_list_last_checked TO installations_last_checked; diff --git a/xmtp_mls/migrations/2024-02-23-174003_add_msg_delivery_status/down.sql b/xmtp_mls/migrations/2024-02-23-174003_add_msg_delivery_status/down.sql deleted file mode 100644 index b1b80b014..000000000 --- a/xmtp_mls/migrations/2024-02-23-174003_add_msg_delivery_status/down.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Copy and replace is only necesasry for SQLite as SQLite does not support DROP COLUMN directly. -BEGIN TRANSACTION; -CREATE TEMPORARY TABLE backup_group(id BLOB PRIMARY KEY NOT NULL, created_at_ns BIGINT NOT NULL, membership_state INT NOT NULL, installations_last_checked BIGINT NOT NULL); -INSERT INTO backup_group SELECT id, created_at_ns, membership_state, installations_last_checked FROM groups; -DROP TABLE groups; -CREATE TABLE groups(id BLOB PRIMARY KEY NOT NULL, created_at_ns BIGINT NOT NULL, membership_state INT NOT NULL, installations_last_checked BIGINT NOT NULL); -INSERT INTO groups SELECT id, created_at_ns, membership_state, installations_last_checked FROM backup_group; -DROP TABLE backup_group; -COMMIT; diff --git a/xmtp_mls/migrations/2024-02-23-174003_add_msg_delivery_status/up.sql b/xmtp_mls/migrations/2024-02-23-174003_add_msg_delivery_status/up.sql deleted file mode 100644 index 344280053..000000000 --- a/xmtp_mls/migrations/2024-02-23-174003_add_msg_delivery_status/up.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Values are: 1 = Published, 2 = Unpublished -ALTER TABLE group_messages -ADD COLUMN "delivery_status" INT NOT NULL DEFAULT 1 diff --git a/xmtp_mls/migrations/2024-03-15-152716_group_types/down.sql b/xmtp_mls/migrations/2024-03-15-152716_group_types/down.sql deleted file mode 100644 index 1c6563fed..000000000 --- a/xmtp_mls/migrations/2024-03-15-152716_group_types/down.sql +++ /dev/null @@ -1,9 +0,0 @@ --- As SQLite does not support ALTER, we play this game of move, repopulate, drop. Here we recreate without the 'purpose' column. -BEGIN TRANSACTION; -CREATE TEMPORARY TABLE backup_group(id BLOB PRIMARY KEY NOT NULL, created_at_ns BIGINT NOT NULL, membership_state INT NOT NULL, installations_last_checked BIGINT NOT NULL); -INSERT INTO backup_group SELECT id, created_at_ns, membership_state, installations_last_checked FROM groups; -DROP TABLE groups; -CREATE TABLE groups(id BLOB PRIMARY KEY NOT NULL, created_at_ns BIGINT NOT NULL, membership_state INT NOT NULL, installations_last_checked BIGINT NOT NULL); -INSERT INTO groups SELECT id, created_at_ns, membership_state, installations_last_checked FROM backup_group; -DROP TABLE backup_group; -COMMIT; diff --git a/xmtp_mls/migrations/2024-03-15-152716_group_types/up.sql b/xmtp_mls/migrations/2024-03-15-152716_group_types/up.sql deleted file mode 100644 index 1defa4b23..000000000 --- a/xmtp_mls/migrations/2024-03-15-152716_group_types/up.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE groups -ADD COLUMN purpose INT NOT NULL DEFAULT 1 diff --git a/xmtp_mls/migrations/2024-04-08-180113_group_added_by_address/down.sql b/xmtp_mls/migrations/2024-04-08-180113_group_added_by_address/down.sql deleted file mode 100644 index 79eca70df..000000000 --- a/xmtp_mls/migrations/2024-04-08-180113_group_added_by_address/down.sql +++ /dev/null @@ -1,9 +0,0 @@ --- As SQLite does not support ALTER, we play this game of move, repopulate, drop. Here we recreate without the 'added_by_address' column. -BEGIN TRANSACTION; -CREATE TEMPORARY TABLE backup_group(id BLOB PRIMARY KEY NOT NULL, created_at_ns BIGINT NOT NULL, membership_state INT NOT NULL, installations_last_checked BIGINT NOT NULL, purpose INT NOT NULL DEFAULT 1); -INSERT INTO backup_group SELECT id, created_at_ns, membership_state, installations_last_checked, pupose FROM groups; -DROP TABLE groups; -CREATE TABLE groups(id BLOB PRIMARY KEY NOT NULL, created_at_ns BIGINT NOT NULL, membership_state INT NOT NULL, installations_last_checked BIGINT NOT NULL, purpose INT NOT NULL DEFAULT 1); -INSERT INTO groups SELECT id, created_at_ns, membership_state, installations_last_checked, purpose FROM backup_group; -DROP TABLE backup_group; -COMMIT; diff --git a/xmtp_mls/migrations/2024-04-08-180113_group_added_by_address/up.sql b/xmtp_mls/migrations/2024-04-08-180113_group_added_by_address/up.sql deleted file mode 100644 index 588747bc8..000000000 --- a/xmtp_mls/migrations/2024-04-08-180113_group_added_by_address/up.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE groups -ADD COLUMN added_by_address TEXT NOT NULL DEFAULT 'placeholder_address'; diff --git a/xmtp_mls/migrations/2024-04-11-004240_identity_init/down.sql b/xmtp_mls/migrations/2024-04-11-004240_identity_init/down.sql deleted file mode 100644 index 8c3077a4d..000000000 --- a/xmtp_mls/migrations/2024-04-11-004240_identity_init/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE identity_updates; \ No newline at end of file diff --git a/xmtp_mls/migrations/2024-04-11-004240_identity_init/up.sql b/xmtp_mls/migrations/2024-04-11-004240_identity_init/up.sql deleted file mode 100644 index 98efbc84b..000000000 --- a/xmtp_mls/migrations/2024-04-11-004240_identity_init/up.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE identity_updates ( - -- The inbox_id the update refers to - "inbox_id" TEXT NOT NULL, - -- The sequence_id of the update - "sequence_id" BIGINT NOT NULL, - -- Based on the timestamp given by the server - "server_timestamp_ns" BIGINT NOT NULL, - -- Random ID generated by group creator - "payload" BLOB NOT NULL, - -- Compound primary key of the `inbox_id` and `sequence_id` - PRIMARY KEY ("inbox_id", "sequence_id") -); - -CREATE INDEX idx_identity_updates_inbox_id_sequence_id_asc ON identity_updates (inbox_id, sequence_id ASC); \ No newline at end of file diff --git a/xmtp_mls/migrations/2024-04-30-035609_identity_inbox/down.sql b/xmtp_mls/migrations/2024-04-30-035609_identity_inbox/down.sql deleted file mode 100644 index 1edcb9982..000000000 --- a/xmtp_mls/migrations/2024-04-30-035609_identity_inbox/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE identity_inbox; \ No newline at end of file diff --git a/xmtp_mls/migrations/2024-04-30-035609_identity_inbox/up.sql b/xmtp_mls/migrations/2024-04-30-035609_identity_inbox/up.sql deleted file mode 100644 index 60e1792a6..000000000 --- a/xmtp_mls/migrations/2024-04-30-035609_identity_inbox/up.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE identity_inbox ( - "inbox_id" TEXT NOT NULL, - "installation_keys" BLOB NOT NULL, - "credential_bytes" BLOB NOT NULL, - rowid INTEGER PRIMARY KEY CHECK (rowid = 1) -- There can only be one identity -); diff --git a/xmtp_mls/migrations/2024-05-15-145138_new_schema/down.sql b/xmtp_mls/migrations/2024-05-15-145138_new_schema/down.sql new file mode 100644 index 000000000..7564d1e1a --- /dev/null +++ b/xmtp_mls/migrations/2024-05-15-145138_new_schema/down.sql @@ -0,0 +1,15 @@ +-- This file should undo anything in `up.sql` +DROP TABLE IF EXISTS "groups"; + +DROP TABLE IF EXISTS group_messages; + +DROP TABLE IF EXISTS refresh_state; + +DROP TABLE IF EXISTS group_intents; + +DROP TABLE IF EXISTS identity_updates; + +DROP TABLE IF EXISTS identity_inbox; + +DROP TABLE IF EXISTS openmls_key_store; + diff --git a/xmtp_mls/migrations/2024-05-15-145138_new_schema/up.sql b/xmtp_mls/migrations/2024-05-15-145138_new_schema/up.sql new file mode 100644 index 000000000..4c0cb3a57 --- /dev/null +++ b/xmtp_mls/migrations/2024-05-15-145138_new_schema/up.sql @@ -0,0 +1,104 @@ +-- Your SQL goes here +CREATE TABLE openmls_key_store( + key_bytes BLOB PRIMARY KEY NOT NULL, + value_bytes BLOB NOT NULL +); + +CREATE TABLE "identity"( + "inbox_id" text NOT NULL, + "installation_keys" BLOB NOT NULL, + "credential_bytes" BLOB NOT NULL, + rowid integer PRIMARY KEY CHECK (rowid = 1) -- There can only be one identity +); + +CREATE TABLE "groups"( + -- Random ID generated by group creator + "id" BLOB PRIMARY KEY NOT NULL, + -- Based on the timestamp of the welcome message + "created_at_ns" bigint NOT NULL, + -- Enum of GROUP_MEMBERSHIP_STATE + "membership_state" int NOT NULL, + -- Last time the installations were checked for the purpose of seeing if any are missing + "installations_last_checked" bigint NOT NULL, + -- Values are 1 = Conversation, 2 = Sync + "purpose" int NOT NULL DEFAULT 1, + -- Which inbox added you to the group + "added_by_inbox_id" text NOT NULL +); + +-- Allow for efficient sorting of groups +CREATE INDEX groups_created_at_idx ON GROUPS (created_at_ns); + +-- This index allows you to filter by membership_state and then created_at_ns +CREATE INDEX groups_membership_state_created_at_idx ON GROUPS (membership_state, created_at_ns); + +-- Successfully processed messages meant to be returned to the user +CREATE TABLE group_messages( + -- Derived via generate_message_id() in SDK, which hashes several inputs + "id" BLOB PRIMARY KEY NOT NULL, + "group_id" BLOB NOT NULL, + -- Message contents after decryption + "decrypted_message_bytes" BLOB NOT NULL, + -- Based on the timestamp of the message + "sent_at_ns" bigint NOT NULL, + -- Enum GROUP_MESSAGE_KIND + "kind" int NOT NULL, + -- Could remove this if we added a table mapping installation_ids to wallet addresses + "sender_installation_id" BLOB NOT NULL, + -- The inbox_id of the sender + "sender_inbox_id" text NOT NULL, + -- Values are: 1 = Published, 2 = Unpublished + "delivery_status" int NOT NULL DEFAULT 1, + FOREIGN KEY (group_id) REFERENCES "groups"(id) +); + +CREATE INDEX group_messages_group_id_sort_idx ON group_messages(group_id, sent_at_ns); + +-- Used to keep track of the last seen message timestamp in a topic +CREATE TABLE refresh_state( + -- E.g. the Id of the group + "entity_id" BLOB NOT NULL, + -- Welcomes or other types + "entity_kind" integer NOT NULL, -- Need to allow for groups and welcomes to be separated, since a malicious client could manipulate their group ID to match someone's installation_id and make a mess + -- Where you are in the topic + "cursor" bigint NOT NULL, + PRIMARY KEY (entity_id, entity_kind) +); + +-- This table is required to retry messages that do not send successfully due to epoch conflicts +CREATE TABLE group_intents( + -- Serial ID auto-generated by the DB + "id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, + -- Enum INTENT_KIND + "kind" int NOT NULL, + "group_id" BLOB NOT NULL, + -- Some sort of serializable blob that can be used to re-try the message if the first attempt failed due to conflict + "data" BLOB NOT NULL, + -- INTENT_STATE, + "state" int NOT NULL, + -- The hash of the encrypted, concrete, form of the message if it was published. + "payload_hash" BLOB UNIQUE, + -- (Optional) data needed for the post-commit flow. For example, welcome messages + "post_commit_data" BLOB, + -- The number of publish attempts + "publish_attempts" int NOT NULL DEFAULT 0, + FOREIGN KEY (group_id) REFERENCES "groups"(id) +); + +CREATE INDEX group_intents_group_id_state ON group_intents(group_id, state); + +CREATE TABLE identity_updates( + -- The inbox_id the update refers to + "inbox_id" text NOT NULL, + -- The sequence_id of the update + "sequence_id" bigint NOT NULL, + -- Based on the timestamp given by the server + "server_timestamp_ns" bigint NOT NULL, + -- Random ID generated by group creator + "payload" BLOB NOT NULL, + -- Compound primary key of the `inbox_id` and `sequence_id` + PRIMARY KEY (inbox_id, sequence_id) +); + +CREATE INDEX idx_identity_updates_inbox_id_sequence_id_asc ON identity_updates(inbox_id, sequence_id ASC); + diff --git a/xmtp_mls/src/groups/mod.rs b/xmtp_mls/src/groups/mod.rs index 509e3e31f..0cdb69071 100644 --- a/xmtp_mls/src/groups/mod.rs +++ b/xmtp_mls/src/groups/mod.rs @@ -407,8 +407,7 @@ impl MlsGroup { sent_at_ns: now, kind: GroupMessageKind::Application, sender_installation_id: self.context.installation_public_key(), - // TODO: Remove this hack - sender_account_address: self.context.inbox_id(), + sender_inbox_id: self.context.inbox_id(), delivery_status: DeliveryStatus::Unpublished, }; group_message.store(&conn)?; @@ -566,7 +565,7 @@ impl MlsGroup { .map_err(GroupError::from) .and_then(|fetch_result| { fetch_result - .map(|group| group.added_by_address.clone()) + .map(|group| group.added_by_inbox_id.clone()) .ok_or_else(|| GroupError::GroupNotFound) }) } diff --git a/xmtp_mls/src/groups/sync.rs b/xmtp_mls/src/groups/sync.rs index b30d9a493..df8076030 100644 --- a/xmtp_mls/src/groups/sync.rs +++ b/xmtp_mls/src/groups/sync.rs @@ -276,7 +276,7 @@ impl MlsGroup { ) -> Result<(), MessageProcessingError> { debug!("[{}] processing private message", self.context.inbox_id()); let decrypted_message = openmls_group.process_message(provider, message)?; - let (sender_account_address, sender_installation_id) = + let (_sender_account_address, sender_installation_id) = validate_message_sender(openmls_group, &decrypted_message, envelope_timestamp_ns)?; match decrypted_message.into_content() { @@ -301,7 +301,8 @@ impl MlsGroup { sent_at_ns: envelope_timestamp_ns as i64, kind: GroupMessageKind::Application, sender_installation_id, - sender_account_address, + // TODO:nm Get real inbox ID + sender_inbox_id: "TODO".to_string(), delivery_status: DeliveryStatus::Published, } .store(provider.conn_ref())? @@ -325,7 +326,8 @@ impl MlsGroup { sent_at_ns: envelope_timestamp_ns as i64, kind: GroupMessageKind::Application, sender_installation_id, - sender_account_address, + // TODO:nm use real inbox ID + sender_inbox_id: "TODO".to_string(), delivery_status: DeliveryStatus::Published, } .store(provider.conn_ref())? @@ -351,7 +353,8 @@ impl MlsGroup { sent_at_ns: envelope_timestamp_ns as i64, kind: GroupMessageKind::Application, sender_installation_id, - sender_account_address, + // TODO:nm use real inbox ID + sender_inbox_id: "TODO".to_string(), delivery_status: DeliveryStatus::Published, } .store(provider.conn_ref())? @@ -548,7 +551,8 @@ impl MlsGroup { sent_at_ns: timestamp_ns as i64, kind: GroupMessageKind::MembershipChange, sender_installation_id, - sender_account_address, + // TODO:nm use real inbox ID + sender_inbox_id: sender_account_address.clone(), delivery_status: DeliveryStatus::Published, }; diff --git a/xmtp_mls/src/storage/encrypted_store/group.rs b/xmtp_mls/src/storage/encrypted_store/group.rs index 5f10ea342..960ad2e82 100644 --- a/xmtp_mls/src/storage/encrypted_store/group.rs +++ b/xmtp_mls/src/storage/encrypted_store/group.rs @@ -34,8 +34,8 @@ pub struct StoredGroup { pub installations_last_checked: i64, /// Enum, [`Purpose`] signifies the group purpose which extends to who can access it. pub purpose: Purpose, - /// The wallet address of who added the user to a group. - pub added_by_address: String, // TODO: This should be Inbox ID. + /// The inbox_id of who added the user to a group. + pub added_by_inbox_id: String, } impl_fetch!(StoredGroup, groups, Vec); @@ -47,7 +47,7 @@ impl StoredGroup { id: ID, created_at_ns: i64, membership_state: GroupMembershipState, - added_by_address: String, + added_by_inbox_id: String, ) -> Self { Self { id, @@ -55,11 +55,12 @@ impl StoredGroup { membership_state, installations_last_checked: 0, purpose: Purpose::Conversation, - added_by_address, + added_by_inbox_id, } } /// Create a new [`Purpose::Sync`] group. This is less common and is used to sync message history. + /// TODO: Set added_by_inbox to your own inbox_id pub fn new_sync_group( id: ID, created_at_ns: i64, @@ -71,7 +72,7 @@ impl StoredGroup { membership_state, installations_last_checked: 0, purpose: Purpose::Sync, - added_by_address: "".into(), + added_by_inbox_id: "".into(), } } } diff --git a/xmtp_mls/src/storage/encrypted_store/group_message.rs b/xmtp_mls/src/storage/encrypted_store/group_message.rs index ced998d52..087e3dd63 100644 --- a/xmtp_mls/src/storage/encrypted_store/group_message.rs +++ b/xmtp_mls/src/storage/encrypted_store/group_message.rs @@ -31,8 +31,8 @@ pub struct StoredGroupMessage { pub kind: GroupMessageKind, /// The ID of the App Installation this message was sent from. pub sender_installation_id: Vec, - /// Network wallet address of the Sender - pub sender_account_address: String, + /// The Inbox ID of the Sender + pub sender_inbox_id: String, /// We optimistically store messages before sending. pub delivery_status: DeliveryStatus, } @@ -208,7 +208,7 @@ mod tests { decrypted_message_bytes: rand_vec(), sent_at_ns: sent_at_ns.unwrap_or(rand_time()), sender_installation_id: rand_vec(), - sender_account_address: "0x0".to_string(), + sender_inbox_id: "0x0".to_string(), kind: kind.unwrap_or(GroupMessageKind::Application), delivery_status: DeliveryStatus::Unpublished, } diff --git a/xmtp_mls/src/storage/encrypted_store/identity.rs b/xmtp_mls/src/storage/encrypted_store/identity.rs index 705f1f6fd..68e36b598 100644 --- a/xmtp_mls/src/storage/encrypted_store/identity.rs +++ b/xmtp_mls/src/storage/encrypted_store/identity.rs @@ -1,4 +1,4 @@ -use crate::storage::encrypted_store::schema::identity_inbox; +use crate::storage::encrypted_store::schema::identity; use diesel::prelude::*; use xmtp_id::InboxId; @@ -11,7 +11,7 @@ use crate::{ /// Identity of this installation /// There can only be one. #[derive(Insertable, Queryable, Debug, Clone)] -#[diesel(table_name = identity_inbox)] +#[diesel(table_name = identity)] pub struct StoredIdentity { pub inbox_id: InboxId, pub installation_keys: Vec, @@ -19,8 +19,8 @@ pub struct StoredIdentity { rowid: Option, } -impl_fetch!(StoredIdentity, identity_inbox); -impl_store!(StoredIdentity, identity_inbox); +impl_fetch!(StoredIdentity, identity); +impl_store!(StoredIdentity, identity); impl StoredIdentity { pub fn new(inbox_id: InboxId, installation_keys: Vec, credential_bytes: Vec) -> Self { diff --git a/xmtp_mls/src/storage/encrypted_store/schema.rs b/xmtp_mls/src/storage/encrypted_store/schema.rs index c39099c33..cf31992fc 100644 --- a/xmtp_mls/src/storage/encrypted_store/schema.rs +++ b/xmtp_mls/src/storage/encrypted_store/schema.rs @@ -21,7 +21,7 @@ diesel::table! { sent_at_ns -> BigInt, kind -> Integer, sender_installation_id -> Binary, - sender_account_address -> Text, + sender_inbox_id -> Text, delivery_status -> Integer, } } @@ -33,21 +33,12 @@ diesel::table! { membership_state -> Integer, installations_last_checked -> BigInt, purpose -> Integer, - added_by_address -> Text, + added_by_inbox_id -> Text, } } diesel::table! { identity (rowid) { - account_address -> Text, - installation_keys -> Binary, - credential_bytes -> Binary, - rowid -> Nullable, - } -} - -diesel::table! { - identity_inbox (rowid) { inbox_id -> Text, installation_keys -> Binary, credential_bytes -> Binary, @@ -87,7 +78,6 @@ diesel::allow_tables_to_appear_in_same_query!( group_messages, groups, identity, - identity_inbox, identity_updates, openmls_key_store, refresh_state, diff --git a/xmtp_mls/update-C72KZAOXqn6bElzI.db3-shm b/xmtp_mls/update-C72KZAOXqn6bElzI.db3-shm new file mode 100644 index 0000000000000000000000000000000000000000..fe9ac2845eca6fe6da8a63cd096d9cf9e24ece10 GIT binary patch literal 32768 zcmeIuAr62r3