Merge pull request #272 from brave/objects_in_chunks

Objects in chunks
brave · Jan 3, 2019 · 56a0fe1 · 56a0fe1
2 parents 249a663 + 329ee80
commit 56a0fe1
Show file tree

Hide file tree

Showing 4 changed files with 136 additions and 5 deletions.
diff --git a/client/requestUtil.js b/client/requestUtil.js
@@ -173,13 +173,16 @@ RequestUtil.prototype.parseAWSResponse = function (bytes) {
  * @param {number=} maxRecords Limit response to a given number of recods. By default the Sync lib will fetch all matching records, which might take a long time. If falsey, fetch all records.
  * @returns {Promise(Array.<Object>)}
  */
-RequestUtil.prototype.list = function (category, startAt, maxRecords) {
+RequestUtil.prototype.list = function (category, startAt, maxRecords, nextContinuationToken) {
   const prefix = `${this.apiVersion}/${this.userId}/${category}`
   let options = {
     MaxKeys: maxRecords || 1000,
     Bucket: this.bucket,
     Prefix: prefix
   }
+  if (nextContinuationToken !== '') {
+    options.ContinuationToken = nextContinuationToken
+  }
   if (startAt) { options.StartAfter = `${prefix}/${startAt}` }
   return this.withRetry(() => {
     if (this.shouldListObject(startAt, category)) {

diff --git a/client/sync.js b/client/sync.js
@@ -22,6 +22,7 @@ var clientUserId = null
 var clientKeys = {}
 var config = {}
 var seed
+var nextContinuationTokens = {}
 
 /**
  * Logs stuff on the visible HTML page.
@@ -105,7 +106,11 @@ const startSync = (requester) => {
       if (!proto.categories[category]) {
         throw new Error(`Unsupported sync category: ${category}`)
       }
-      requester.list(proto.categories[category], startAt, limitResponse).then((s3Objects) => {
+      let continuationToken = ''
+      if (nextContinuationTokens[category]) {
+        continuationToken = nextContinuationTokens[category]
+      }
+      requester.list(proto.categories[category], startAt, limitResponse, continuationToken).then((s3Objects) => {
         const jsRecords = getJSRecords(s3Objects.contents)
         logSync(`got ${jsRecords.length} decrypted records in ${category} after ${startAt}`)
         let lastRecordTimestamp
@@ -117,6 +122,12 @@ const startSync = (requester) => {
         } else if (!s3Objects.isTruncated) {
           requester.setListInProgress(false)
         }
+        if (s3Objects.isTruncated) {
+          // When is it truncated we need to provide continuation token, so system could understand where to continue from next time
+          nextContinuationTokens[category] = s3Objects.nextContinuationToken
+        } else {
+          nextContinuationTokens[category] = ''
+        }
         ipc.send(messages.GET_EXISTING_OBJECTS, category, jsRecords, lastRecordTimestamp, s3Objects.isTruncated)
       })
     })

diff --git a/lib/s3Helper.js b/lib/s3Helper.js
@@ -221,7 +221,8 @@ module.exports.listObjects = function (s3, options, limitResponse) {
         } else {
           resolve({
             contents: data.Contents,
-            isTruncated: data.IsTruncated
+            isTruncated: data.IsTruncated,
+            nextContinuationToken: data.NextContinuationToken
           })
         }
       })
@@ -230,7 +231,8 @@ module.exports.listObjects = function (s3, options, limitResponse) {
         if (error) { reject(error) }
         resolve({
           contents: data,
-          isTruncated: false
+          isTruncated: false,
+          nextContinuationToken: ''
         })
       })
     }

diff --git a/test/client/requestUtil.js b/test/client/requestUtil.js
@@ -392,16 +392,131 @@ test('client RequestUtil', (t) => {
 
     const testCanLimitResponseToOne = (t) => {
       t.test('limitResponse to 1', (t) => {
-        t.plan(2)
+        t.plan(3)
         requestUtil.list(proto.categories.PREFERENCES, 0, 1)
           .then((s3Objects) => {
             t.assert(s3Objects.isTruncated === true, `${t.name} has true isTruncated value`)
             t.assert(s3Objects.contents.length === 1, `${t.name} has one record`)
+            testCanGetBookmarksInChunks(t)
           })
           .catch((error) => t.fail(error))
       })
     }
 
+    const testCanGetBookmarksInChunks = (t) => {
+      const records = [
+        {
+        action: 'CREATE',
+        deviceId: new Uint8Array([0]),
+        objectId: testHelper.newUuid(),
+        bookmark: {
+          site: {
+            location: `https://brave.com?q=1`,
+            title: 'lulz',
+            lastAccessedTime: 1480000000 * 1000,
+            creationTime: 1480000000 * 1000
+          },
+          isFolder: false,
+          hideInToolbar: false,
+          order: '1.0.0.1'
+        }},
+        {
+          action: 'CREATE',
+          deviceId: new Uint8Array([0]),
+          objectId: testHelper.newUuid(),
+          bookmark: {
+            site: {
+              location: `https://brave.com?q=2`,
+              title: 'lulz',
+              lastAccessedTime: 1480000000 * 1000,
+              creationTime: 1480000000 * 1000
+            },
+            isFolder: false,
+            hideInToolbar: false,
+            order: '1.0.0.2'
+        }},
+        {
+          action: 'CREATE',
+          deviceId: new Uint8Array([0]),
+          objectId: testHelper.newUuid(),
+          bookmark: {
+            site: {
+              location: `https://brave.com?q=3`,
+              title: 'lulz',
+              lastAccessedTime: 1480000000 * 1000,
+              creationTime: 1480000000 * 1000
+            },
+            isFolder: false,
+            hideInToolbar: false,
+            order: '1.0.0.3'
+        }},
+        {
+          action: 'CREATE',
+          deviceId: new Uint8Array([0]),
+          objectId: testHelper.newUuid(),
+          bookmark: {
+            site: {
+              location: `https://brave.com?q=4`,
+              title: 'lulz',
+              lastAccessedTime: 1480000000 * 1000,
+              creationTime: 1480000000 * 1000
+            },
+            isFolder: false,
+            hideInToolbar: false,
+            order: '1.0.0.4'
+        }},
+        {
+          action: 'CREATE',
+          deviceId: new Uint8Array([0]),
+          objectId: testHelper.newUuid(),
+          bookmark: {
+            site: {
+              location: `https://brave.com?q=5`,
+              title: 'lulz',
+              lastAccessedTime: 1480000000 * 1000,
+              creationTime: 1480000000 * 1000
+            },
+            isFolder: false,
+            hideInToolbar: false,
+            order: '1.0.0.5'
+        }}
+      ]
+
+      records.forEach((record) => {
+        requestUtil.put(proto.categories.BOOKMARKS, record)
+      })
+
+      t.test('#getBookmarksInChunks', (t) => {
+        t.plan(1)
+        getBookmarksInChunks(t, '', 1)
+      })
+
+      const getBookmarksInChunks = (t, continuationToken, iterationNumber) => {
+        if ((continuationToken === undefined || continuationToken === '') && iterationNumber > 1) {
+          t.assert(true, 'getBookmarksInChunks exit recurtion')
+          return
+        }
+        t.test('#getBookmarksInChunks attempt #' + iterationNumber, (t) => {
+          t.plan(3)
+          requestUtil.list(proto.categories.BOOKMARKS, 0, 3, continuationToken)
+          .then((s3Objects) => {
+            t.assert(s3Objects.contents.length <= 3, `${t.name} has less or exactly 3 records`)
+            if (s3Objects.isTruncated === true && s3Objects.nextContinuationToken !== '' && s3Objects.nextContinuationToken !== undefined) {
+              t.assert(true, `${t.name} isTruncated is true and nextContinuationToken is not empty`)
+            } else if (s3Objects.isTruncated === false && (s3Objects.nextContinuationToken === '' || s3Objects.nextContinuationToken === undefined) && iterationNumber > 1) {
+              t.assert(true, `${t.name} isTruncated is false and nextContinuationToken is empty`)
+            } else {
+              t.assert(false, `${t.name} isTruncated and nextContinuationToken values doesn't match, iterationNumber: ${iterationNumber}`)
+            }
+            continuationToken = s3Objects.nextContinuationToken
+            iterationNumber = iterationNumber + 1
+            getBookmarksInChunks(t, continuationToken, iterationNumber)
+          })
+          .catch((error) => t.fail(error))
+        })
+      }
+    }
+
     const expiredCredentials = {
       aws: clientTestHelper.EXPIRED_CREDENTIALS.aws,
       s3Post: clientTestHelper.EXPIRED_CREDENTIALS.s3Post,