feat(core): long running operation recovery (#882)

* feat: basic record to mark restore as complete * feat: mark done when finished recovering in the DB * feat: update unittest * update: resolve comment * update: update unit test * feat: resolve comment * feat: long running operation recovery * remove duplicate logic * feat(core): recovery of individual IPEX on-going messages and general recovery method * refactor: remove comment (ticket created) * test(core): recovery of ipex exns --------- Co-authored-by: iFergal <[email protected]>
cardano-foundation · Jan 3, 2025 · ecb326c · ecb326c
1 parent 5ac7c4f
commit ecb326c
Show file tree

Hide file tree

Showing 19 changed files with 488 additions and 354 deletions.
diff --git a/src/core/agent/agent.test.ts b/src/core/agent/agent.test.ts
@@ -46,10 +46,13 @@ const mockCredentialService = {
   syncKeriaCredentials: jest.fn(),
   removeCredentialsPendingDeletion: jest.fn(),
 };
+const mockKeriaNotificationService = {
+  syncIPEXReplyOperations: jest.fn(),
+};
 
 const mockEntropy = "00000000000000000000000000000000";
 
-describe("test cases of bootAndConnect function", () => {
+describe("KERIA connectivity", () => {
   let agent: Agent;
   let mockAgentUrls: AgentUrls;
   let mockSignifyClient: any;
@@ -238,7 +241,7 @@ describe("test cases of bootAndConnect function", () => {
   });
 });
 
-describe("test cases of recoverKeriaAgent function", () => {
+describe("Recovery of DB from cloud sync", () => {
   let agent: Agent;
   let mockSeedPhrase: string[];
   let mockConnectUrl: string;
@@ -254,6 +257,7 @@ describe("test cases of recoverKeriaAgent function", () => {
     (agent as any).basicStorageService = mockBasicStorageService;
     (agent as any).agentServicesProps = mockAgentServicesProps;
     (agent as any).connectionService = mockConnectionService;
+    (agent as any).keriaNotificationService = mockKeriaNotificationService;
 
     mockSeedPhrase = [
       "abandon",
@@ -306,6 +310,7 @@ describe("test cases of recoverKeriaAgent function", () => {
     expect(mockConnectionService.syncKeriaContacts).toHaveBeenCalled();
     expect(mockIdentifierService.syncKeriaIdentifiers).toHaveBeenCalled();
     expect(mockCredentialService.syncKeriaCredentials).toHaveBeenCalled();
+    expect(mockKeriaNotificationService.syncIPEXReplyOperations).toHaveBeenCalled();
     expect(mockSignifyClient.connect).toHaveBeenCalled();
     expect(mockBasicStorageService.createOrUpdateBasicRecord).toHaveBeenCalledWith({
       _tags: {},
@@ -319,13 +324,6 @@ describe("test cases of recoverKeriaAgent function", () => {
       KeyStoreKeys.SIGNIFY_BRAN,
       expectedBran
     );
-    expect(Agent.isOnline).toBe(true);
-    expect(mockAgentServicesProps.eventEmitter.emit).toBeCalledWith({
-      type: EventTypes.KeriaStatusChanged,
-      payload: {
-        isOnline: true,
-      },
-    });
   });
 
   test("should throw an error for invalid mnemonic", async () => {

diff --git a/src/core/agent/agent.ts b/src/core/agent/agent.ts
@@ -314,15 +314,14 @@ class Agent {
     await this.connections.syncKeriaContacts();
     await this.identifiers.syncKeriaIdentifiers();
     await this.credentials.syncKeriaCredentials();
+    await this.keriaNotifications.syncIPEXReplyOperations();
 
     await this.basicStorage.createOrUpdateBasicRecord(
       new BasicRecord({
         id: MiscRecordId.CLOUD_RECOVERY_STATUS,
         content: { syncing: false },
       })
     );
-
-    this.markAgentStatus(true);
   }
 
   private async connectSignifyClient(): Promise<void> {
@@ -349,6 +348,7 @@ class Agent {
     });
   }
 
+  // For now this is called by UI/AppWrapper to not prematurely mark online while mid-recovery
   markAgentStatus(online: boolean) {
     Agent.isOnline = online;
 

diff --git a/src/core/agent/event.types.ts b/src/core/agent/event.types.ts
@@ -29,7 +29,7 @@ enum EventTypes {
 interface NotificationAddedEvent extends BaseEventEmitter {
   type: typeof EventTypes.NotificationAdded;
   payload: {
-    keriaNotif: KeriaNotification;
+    note: KeriaNotification;
   };
 }
 

diff --git a/src/core/agent/records/notificationRecord.ts b/src/core/agent/records/notificationRecord.ts
@@ -1,6 +1,6 @@
 import { BaseRecord, Tags } from "../../storage/storage.types";
 import { NotificationRoute } from "../agent.types";
-import { LinkedGroupRequest } from "./notificationRecord.types";
+import { LinkedRequest } from "./notificationRecord.types";
 import { randomSalt } from "../services/utils";
 
 interface NotificationRecordStorageProps {
@@ -13,7 +13,7 @@ interface NotificationRecordStorageProps {
   multisigId?: string;
   connectionId: string;
   credentialId?: string;
-  linkedGroupRequest?: LinkedGroupRequest;
+  linkedRequest?: LinkedRequest;
   groupReplied?: boolean,
   initiatorAid?: string,
   groupInitiator?: boolean,
@@ -25,11 +25,12 @@ class NotificationRecord extends BaseRecord {
   read!: boolean;
   multisigId?: string;
   connectionId!: string;
-  linkedGroupRequest!: LinkedGroupRequest;
+  linkedRequest!: LinkedRequest;
   credentialId?: string;
   groupReplied?: boolean;
   initiatorAid?: string;
   groupInitiator?: boolean;
+  hidden = false;  // Hide from UI but don't delete (used for reliability while recovering IPEX long running operations)
 
   static readonly type = "NotificationRecord";
   readonly type = NotificationRecord.type;
@@ -45,7 +46,7 @@ class NotificationRecord extends BaseRecord {
       this.multisigId = props.multisigId;
       this.connectionId = props.connectionId;
       this._tags = props.tags ?? {};
-      this.linkedGroupRequest = props.linkedGroupRequest ?? { accepted: false };
+      this.linkedRequest = props.linkedRequest ?? { accepted: false };
       this.credentialId = props.credentialId;
       this.groupReplied = props.groupReplied;
       this.initiatorAid = props.initiatorAid;
@@ -55,6 +56,7 @@ class NotificationRecord extends BaseRecord {
 
   getTags() {
     return {
+      ...this._tags,
       route: this.route,
       read: this.read,
       multisigId: this.multisigId,
@@ -63,7 +65,8 @@ class NotificationRecord extends BaseRecord {
       groupReplied: this.groupReplied,
       initiatorAid: this.initiatorAid,
       groupInitiator: this.groupInitiator,
-      ...this._tags,
+      currentLinkedRequest: this.linkedRequest.current,
+      hidden: this.hidden,
     };
   }
 }

diff --git a/src/core/agent/records/notificationRecord.types.ts b/src/core/agent/records/notificationRecord.types.ts
@@ -1,6 +1,6 @@
 import { Notification } from "../services/credentialService.types";
 
-interface LinkedGroupRequest {
+interface LinkedRequest {
   accepted: boolean;
   current?: string;
   previous?: string;
@@ -12,4 +12,4 @@ interface NotificationAttempts {
   notification: Notification;
 }
 
-export type { LinkedGroupRequest, NotificationAttempts };
+export type { LinkedRequest, NotificationAttempts };
diff --git a/src/core/agent/services/identifierService.ts b/src/core/agent/services/identifierService.ts
@@ -502,18 +502,9 @@ class IdentifierService extends AgentService {
 
       const op: Operation = await this.props.signifyClient
         .operations()
-        .get(`witness.${identifier.prefix}`)
-        .catch(async (error) => {
-          const status = error.message.split(" - ")[1];
-          if (/404/gi.test(status)) {
-            return await this.props.signifyClient
-              .operations()
-              .get(`done.${identifier.prefix}`);
-          }
-          throw error;
-        });
+        .get(`witness.${identifier.prefix}`);
+
       const isPending = !op.done;
-
       if (isPending) {
         const pendingOperation = await this.operationPendingStorage.save({
           id: op.name,
@@ -531,6 +522,7 @@ class IdentifierService extends AgentService {
       const identifierDetail = (await this.props.signifyClient
         .identifiers()
         .get(identifier.prefix)) as HabState & { icp_dt: string };
+
       if (isMultiSig) {
         const groupId = identifier.name.split(":")[1];
         const groupInitiator = groupId.split("-")[0] === "1";
@@ -563,19 +555,21 @@ class IdentifierService extends AgentService {
       if (identifier.name.startsWith("XX")) {
         continue;
       }
+
+      const identifierDetail = (await this.props.signifyClient
+        .identifiers()
+        .get(identifier.prefix)) as HabState & { icp_dt: string };
 
       const multisigManageAid = identifier.group.mhab.prefix;
       const groupId = identifier.group.mhab.name.split(":")[1];
       const theme = parseInt(identifier.name.split(":")[0], 10);
       const groupInitiator = groupId.split("-")[0] === "1";
+
       const op = await this.props.signifyClient
         .operations()
         .get(`group.${identifier.prefix}`);
+
       const isPending = !op.done;
-      const identifierDetail = (await this.props.signifyClient
-        .identifiers()
-        .get(identifier.prefix)) as HabState & { icp_dt: string };
-
       if (isPending) {
         const pendingOperation = await this.operationPendingStorage.save({
           id: op.name,