feat: fetch only recent messages; fetch in batches

packages/main/src/cache/query.ts

@@ -2,6 +2,22 @@ import imap from "imap"
 import db from "../db"
 import { ID, Message } from "./types"
 
+export function firstSeenUid(accountId: ID, box: { name: string }): number {
+  const boxRecord = getBoxRecord(accountId, box)
+  if (!boxRecord) {
+    return 0
+  }
+  const result: { max_uid: number | null } = db
+    .prepare(
+      `
+        select min(uid) as max_uid from messages
+        where box_id = ?
+      `
+    )
+    .get(boxRecord.id)
+  return result.max_uid || 0
+}
+
 export function lastSeenUid(accountId: ID, box: { name: string }): number {
   const boxRecord = getBoxRecord(accountId, box)
   if (!boxRecord) {
@@ -168,9 +184,16 @@ export function getBody(
   return result && result.content
 }
 
-export function partsMissingBodies(params: {
+export function partsMissingBodies({
+  accountId,
+  boxId,
+  minUid = 0,
+  maxUid = 0
+}: {
   accountId: ID
   boxId: ID
+  minUid?: number
+  maxUid?: number
 }): Array<{ uid: number; boxName: string; part: imap.ImapMessagePart }> {
   return db
     .prepare(
@@ -186,9 +209,11 @@ export function partsMissingBodies(params: {
           and messages.account_id = @accountId
           and bodies.content is null
           and structs.rgt = structs.lft + 1
+          and messages.uid >= @minUid
+          and (@maxUid <= 0 or messages.uid <= @maxUid)
       `
     )
-    .all(params)
+    .all({ accountId, boxId, minUid, maxUid })
     .map(row => ({
       uid: row.uid,
       boxName: row.boxName,

packages/main/src/cache/testFixtures.ts

@@ -14,6 +14,22 @@ export const inbox: imap.Box = {
   readOnly: true
 }
 
+export const allMail: imap.Box = {
+  name: "[Gmail]/All Mail",
+  readOnly: true,
+  uidvalidity: 123,
+  uidnext: 7688,
+  flags: [],
+  permFlags: [],
+  newKeywords: false,
+  persistentUIDs: true,
+  messages: {
+    total: 2,
+    new: 0,
+    unseen: 0
+  }
+}
+
 export const testThread: Array<{
   attributes: imap.ImapMessageAttributes
   headers: Array<[string, HeaderValue]>

packages/main/src/queue/index.test.ts

@@ -1,6 +1,6 @@
 import Connection from "imap"
 import * as cache from "../cache"
-import { inbox, testThread } from "../cache/testFixtures"
+import { allMail, testThread } from "../cache/testFixtures"
 import db from "../db"
 import AccountManager from "../managers/AccountManager"
 import { mockConnection } from "../request/testHelpers"
@@ -40,7 +40,7 @@ it("marks a conversation as read", async () => {
     .enqueue(
       queue.actions.markAsRead({
         accountId: String(accountId),
-        box: inbox,
+        box: allMail,
         uids: [7687]
       })
     )
@@ -72,7 +72,7 @@ it("marks a conversation as unread", async () => {
     .enqueue(
       queue.actions.unmarkAsRead({
         accountId: String(accountId),
-        box: inbox,
+        box: allMail,
         uids: [7687]
       })
     )
@@ -105,7 +105,7 @@ it.skip("replaces pending read status change when a new change is queued", async
     .enqueue(
       queue.actions.markAsRead({
         accountId: String(accountId),
-        box: inbox,
+        box: allMail,
         uids: [7687]
       })
     )
@@ -114,7 +114,7 @@ it.skip("replaces pending read status change when a new change is queued", async
     .enqueue(
       queue.actions.unmarkAsRead({
         accountId: String(accountId),
-        box: inbox,
+        box: allMail,
         uids: [7687]
       })
     )

packages/main/src/queue/index.ts

@@ -108,14 +108,10 @@ const store =
       })
 
 export const queue = new BetterQueue<Task>(processTask, {
-  maxRetries: Infinity, // keep retrying until we come online
+  maxRetries: 3,
   maxTimeout: process.env.NODE_ENV === "test" ? 150 : 10000,
   retryDelay: process.env.NODE_ENV === "test" ? 1 : 10000,
-  store,
-  merge(_oldTask, newTask, cb) {
-    console.log("merge", _oldTask, newTask)
-    cb(null, newTask)
-  }
+  store
 })
 
 export function getQueuedTasks(): Array<Action<unknown>> {

packages/main/src/request/index.ts

@@ -238,3 +238,9 @@ export function isHeaders(x: FetchResponse): x is Headers {
 export function isMessage(x: FetchResponse): x is Message {
   return x.type === MESSAGE
 }
+
+export function messageAttributes(
+  x: FetchResponse
+): imap.ImapMessageAttributes {
+  return isMessage(x) ? x.attributes : x.messageAttributes
+}

packages/main/src/request/testHelpers.ts

@@ -21,7 +21,8 @@ export function mockConnection({
   thread?: Message[]
 } = {}): ConnectionManager {
   const boxes = {
-    INBOX: { attribs: ["\\Inbox"] }
+    INBOX: { attribs: ["\\Inbox"] },
+    "[Gmail]/All Mail": { attribs: ["\\All"] }
   }
 
   mock(Connection.prototype.addFlags).mockImplementation(
@@ -56,7 +57,7 @@ export function mockConnection({
       name,
       readOnly,
       uidvalidity: 123,
-      uidnext: 456,
+      uidnext: testThread[1].attributes.uid + 1,
       flags: [],
       permFlags: [],
       newKeywords: false,

packages/main/src/resolvers/conversation.test.ts

@@ -4,10 +4,12 @@ import * as cache from "../cache"
 import { testThread } from "../cache/testFixtures"
 import db from "../db"
 import ConnectionManager from "../managers/ConnectionManager"
+import { queue } from "../queue"
 import { mockConnection, mockFetchImplementation } from "../request/testHelpers"
 import schema from "../schema"
 import { sync } from "../sync"
 import { mock } from "../testHelpers"
+import * as promises from "../util/promises"
 
 jest.mock("imap")
 
@@ -223,6 +225,7 @@ it("marks a conversation as unread", async () => {
   })
 })
 
-afterEach(() => {
+afterEach(async () => {
   db.prepare("delete from accounts").run()
+  await promises.lift0(cb => queue.destroy(cb))
 })

packages/main/src/sync.test.ts

@@ -1,9 +1,9 @@
 import Connection from "imap"
 import * as cache from "./cache"
-import { inbox, testThread } from "./cache/testFixtures"
+import { testThread } from "./cache/testFixtures"
 import db from "./db"
 import ConnectionManager from "./managers/ConnectionManager"
-import { mockFetchImplementation } from "./request/testHelpers"
+import { mockConnection, mockFetchImplementation } from "./request/testHelpers"
 import { sync } from "./sync"
 import { mock } from "./testHelpers"
 
@@ -18,40 +18,22 @@ beforeEach(() => {
     .run("[email protected]")
   accountId = lastInsertRowid
 
-  const boxes = {
-    INBOX: {
-      attribs: ["\\Inbox"],
-      uidnext: cache.lastSeenUid(accountId, { name: "INBOX" }) + 1
-    }
-  }
-  mock(Connection.prototype.getBoxes).mockImplementation((cb: any) => {
-    cb(null, boxes)
-  })
-  mock(Connection.prototype.fetch).mockImplementation(mockFetchImplementation())
-  mock(Connection.prototype.connect).mockReturnValue(undefined)
-
-  connectionManager = new ConnectionManager(async () => {
-    const conn = new Connection({})
-    conn.state = "connected"
-    ;(conn as any)._box = inbox
-    return conn
-  })
+  connectionManager = mockConnection()
 })
 
-it("records metadata for INBOX", async () => {
+it("records metadata for mailbox", async () => {
   await sync(accountId, connectionManager)
   expect(db.prepare("select * from boxes").all()).toMatchObject([
-    { account_id: accountId, name: "INBOX", uidvalidity: 9999 }
+    { account_id: accountId, name: "[Gmail]/All Mail", uidvalidity: 123 }
   ])
 })
 
-it("downloads messages from INBOX", async () => {
+it("downloads messages", async () => {
   await sync(accountId, connectionManager)
   const messages = db
     .prepare(
       `
-        select boxes.account_id, messages.uid from messages
-        join boxes on messages.box_id = boxes.id
+        select account_id, uid from messages
       `
     )
     .all()
@@ -121,14 +103,14 @@ it("gets updated labels for messages that are already downloaded", async () => {
         {
           attributes: {
             ...testThread[0].attributes,
-            "x-gm-labels": ["\\Important", "\\Sent"]
+            "x-gm-labels": ["\\Inbox", "\\Sent"]
           },
           headers: testThread[0].headers
         },
         {
           attributes: {
             ...testThread[1].attributes,
-            "x-gm-labels": ["\\Sent"]
+            "x-gm-labels": ["\\Inbox", "\\Sent", "Followup"]
           },
           headers: testThread[1].headers
         }
@@ -149,8 +131,10 @@ it("gets updated labels for messages that are already downloaded", async () => {
       )
       .all()
   ).toMatchObject([
-    { uid: 7467, label: "\\Important" },
+    { uid: 7467, label: "\\Inbox" },
     { uid: 7467, label: "\\Sent" },
+    { uid: 7687, label: "Followup" },
+    { uid: 7687, label: "\\Inbox" },
     { uid: 7687, label: "\\Sent" }
   ])
 })