restore: Auto-clean old blobs from IndexedDB

CHANGELOG.md

@@ -101,6 +101,7 @@ To be released: 2017-10-27
 - [ ] plugins: add tabindex="0" to buttons and tabs?
 - [ ] goldenretriever: add “ghost” files (@arturi)
 - [ ] xhrupload: set a timeout in the onprogress event handler to detect stale network
+- [ ] goldenretriever: add expiration to ServiceWorkerStore (@goto-bus-stop)
 
 # next
 
@@ -120,7 +121,7 @@ Theme: React and Retry
 - [x] core: fix `replaceTargetContent` and add tests for `Plugin` (#354 / @gavboulton)
 - [x] goldenretriever: Omit completed uploads from saved file state—previously, when an upload was finished and the user refreshed the page, all the finished files would still be there because we saved the entire list of files. Changed this to only store files that are part of an in-progress upload, or that have yet to be uploaded (#358, #324 / @goto-bus-stop)
 - [x] goldenretriever: Remove files from cache when upload finished—this uses the deleteBlobs function when core:success fires (#358, #324 / @goto-bus-stop)
-- [ ] goldenretriever: add a timestamp to cached blobs, and to delete old blobs on boot (#358, #324 / @goto-bus-stop)
+- [x] goldenretriever: add a timestamp to cached blobs, and to delete old blobs on boot (#358, #324 / @goto-bus-stop)
 - [x] s3: have some way to configure content-disposition for uploads, see #243 (@goto-bus-stop)
 - [x] core: move `setPluginState` and add `getPluginState` to `Plugin` class (#363 / @goto-bus-stop)
 

examples/bundled-example/main.js

@@ -76,20 +76,11 @@ uppy.on('core:success', (fileList) => {
   console.log(fileList)
 })
 
-const isServiceWorkerControllerReady = new Promise(resolve => {
-  if (navigator.serviceWorker.controller) return resolve()
-  navigator.serviceWorker.addEventListener('controllerchange', e => resolve())
-})
-
 if ('serviceWorker' in navigator) {
   navigator.serviceWorker
     .register('/sw.js')
     .then((registration) => {
       console.log('ServiceWorker registration successful with scope: ', registration.scope)
-      return isServiceWorkerControllerReady
-    })
-    .then(() => {
-      uppy.emit('core:file-sw-ready')
     })
     .catch((error) => {
       console.log('Registration failed with ' + error)

src/plugins/RestoreFiles/IndexedDBStore.js

@@ -1,19 +1,49 @@
+const prettyBytes = require('prettier-bytes')
 const indexedDB = window.indexedDB || window.webkitIndexedDB || window.mozIndexedDB || window.OIndexedDB || window.msIndexedDB
 
 const isSupported = !!indexedDB
 
 const DB_NAME = 'uppy-blobs'
 const STORE_NAME = 'files' // maybe have a thumbnail store in the future
-const DB_VERSION = 2
+const DEFAULT_EXPIRY = 24 * 60 * 60 * 1000 // 24 hours
+const DB_VERSION = 3
+
+// Set default `expires` dates on existing stored blobs.
+function migrateExpiration (store) {
+  const request = store.openCursor()
+  request.onsuccess = (event) => {
+    const cursor = event.target.result
+    if (!cursor) {
+      return
+    }
+    const entry = cursor.value
+    entry.expires = Date.now() + DEFAULT_EXPIRY
+    cursor.update(entry)
+  }
+}
 
 function connect (dbName) {
   const request = indexedDB.open(dbName, DB_VERSION)
   return new Promise((resolve, reject) => {
     request.onupgradeneeded = (event) => {
       const db = event.target.result
-      const store = db.createObjectStore(STORE_NAME, { keyPath: 'id' })
-      store.createIndex('store', 'store', { unique: false })
-      store.transaction.oncomplete = () => {
+      const transaction = event.currentTarget.transaction
+
+      if (event.oldVersion < 2) {
+        // Added in v2: DB structure changed to a single shared object store
+        const store = db.createObjectStore(STORE_NAME, { keyPath: 'id' })
+        store.createIndex('store', 'store', { unique: false })
+      }
+
+      if (event.oldVersion < 3) {
+        // Added in v3
+        const store = transaction.objectStore(STORE_NAME)
+        store.createIndex('expires', 'expires', { unique: false })
+
+        migrateExpiration(store)
+      }
+
+      transaction.oncomplete = () => {
         resolve(db)
       }
     }
@@ -33,17 +63,30 @@ function waitForRequest (request) {
   })
 }
 
+let cleanedUp = false
 class IndexedDBStore {
-  constructor (core, opts) {
+  constructor (opts) {
     this.opts = Object.assign({
       dbName: DB_NAME,
       storeName: 'default',
+      expires: DEFAULT_EXPIRY, // 24 hours
       maxFileSize: 10 * 1024 * 1024, // 10 MB
       maxTotalSize: 300 * 1024 * 1024 // 300 MB
     }, opts)
 
     this.name = this.opts.storeName
-    this.ready = connect(this.opts.dbName)
+
+    const createConnection = () => {
+      return connect(this.opts.dbName)
+    }
+
+    if (!cleanedUp) {
+      cleanedUp = true
+      this.ready = IndexedDBStore.cleanup()
+        .then(createConnection, createConnection)
+    } else {
+      this.ready = createConnection
+    }
   }
 
   key (fileID) {
@@ -131,6 +174,7 @@ class IndexedDBStore {
         id: this.key(file.id),
         fileID: file.id,
         store: this.name,
+        expires: Date.now() + this.opts.expires,
         data: file.data
       })
       return waitForRequest(request)
@@ -148,6 +192,38 @@ class IndexedDBStore {
       return waitForRequest(request)
     })
   }
+
+  /**
+   * Delete all stored blobs that have an expiry date that is before Date.now().
+   * This is a static method because it deletes expired blobs from _all_ Uppy instances.
+   */
+  static cleanup () {
+    return connect(DB_NAME).then((db) => {
+      const transaction = db.transaction([STORE_NAME], 'readwrite')
+      const store = transaction.objectStore(STORE_NAME)
+      const request = store.index('expires')
+        .openCursor(IDBKeyRange.upperBound(Date.now()))
+      return new Promise((resolve, reject) => {
+        request.onsuccess = (event) => {
+          const cursor = event.target.result
+          if (cursor) {
+            const entry = cursor.value
+            console.log(
+              '[IndexedDBStore] Deleting record', entry.fileID,
+              'of size', prettyBytes(entry.data.size),
+              '- expired on', new Date(entry.expires))
+            cursor.delete() // Ignoring return value … it's not terrible if this goes wrong.
+            cursor.continue()
+          } else {
+            resolve(db)
+          }
+        }
+        request.onerror = reject
+      })
+    }).then((db) => {
+      db.close()
+    })
+  }
 }
 
 IndexedDBStore.isSupported = isSupported

src/plugins/RestoreFiles/MetaDataStore.js

@@ -0,0 +1,85 @@
+/**
+ * Get uppy instance IDs for which state is stored.
+ */
+function findUppyInstances () {
+  const instances = []
+  for (let i = 0; i < localStorage.length; i++) {
+    const key = localStorage.key(i)
+    if (/^uppyState:/.test(key)) {
+      instances.push(key.slice('uppyState:'.length))
+    }
+  }
+  return instances
+}
+
+/**
+ * Try to JSON-parse a string, return null on failure.
+ */
+function maybeParse (str) {
+  try {
+    return JSON.parse(str)
+  } catch (err) {
+    return null
+  }
+}
+
+let cleanedUp = false
+module.exports = class MetaDataStore {
+  constructor (opts) {
+    this.opts = Object.assign({
+      expires: 24 * 60 * 60 * 1000 // 24 hours
+    }, opts)
+    this.name = `uppyState:${opts.storeName}`
+
+    if (!cleanedUp) {
+      cleanedUp = true
+      MetaDataStore.cleanup()
+    }
+  }
+
+  /**
+   *
+   */
+  load () {
+    const savedState = localStorage.getItem(this.name)
+    if (!savedState) return null
+    const data = maybeParse(savedState)
+    if (!data) return null
+
+    // Upgrade pre-0.20.0 uppyState: it used to be just a flat object,
+    // without `expires`.
+    if (!data.metadata) {
+      this.save(data)
+      return data
+    }
+
+    return data.metadata
+  }
+
+  save (metadata) {
+    const expires = Date.now() + this.opts.expires
+    const state = JSON.stringify({
+      metadata,
+      expires
+    })
+    localStorage.setItem(this.name, state)
+  }
+
+  /**
+   * Remove all expired state.
+   */
+  static cleanup () {
+    const instanceIDs = findUppyInstances()
+    const now = Date.now()
+    instanceIDs.forEach((id) => {
+      const data = localStorage.getItem(`uppyState:${id}`)
+      if (!data) return null
+      const obj = maybeParse(data)
+      if (!obj) return null
+
+      if (obj.expires && obj.expires < now) {
+        localStorage.removeItem(`uppyState:${id}`)
+      }
+    })
+  }
+}

src/plugins/RestoreFiles/ServiceWorkerStore.js

@@ -1,13 +1,23 @@
 const isSupported = 'serviceWorker' in navigator
 
-class ServiceWorkerStore {
-  constructor (core, opts) {
-    this.core = core
-    this.ready = new Promise((resolve, reject) => {
-      this.core.on('core:file-sw-ready', () => {
+function waitForServiceWorker () {
+  return new Promise((resolve, reject) => {
+    if (!('serviceWorker' in navigator)) {
+      reject(new Error('Unsupported'))
+    } else if (navigator.serviceWorker.controller) {
+      // A serviceWorker is already registered and active.
+      resolve()
+    } else {
+      navigator.serviceWorker.addEventListener('controllerchange', () => {
         resolve()
       })
-    })
+    }
+  })
+}
+
+class ServiceWorkerStore {
+  constructor (opts) {
+    this.ready = waitForServiceWorker()
     this.name = opts.storeName
   }
 

src/plugins/RestoreFiles/cleanup.js

@@ -0,0 +1,10 @@
+const IndexedDBStore = require('./IndexedDBStore')
+const MetaDataStore = require('./MetaDataStore')
+
+/**
+ * Clean old blobs without needing to import all of Uppy.
+ */
+module.exports = function cleanup () {
+  MetaDataStore.cleanup()
+  IndexedDBStore.cleanup()
+}

src/plugins/RestoreFiles/index.js

@@ -1,6 +1,7 @@
 const Plugin = require('../Plugin')
 const ServiceWorkerStore = require('./ServiceWorkerStore')
 const IndexedDBStore = require('./IndexedDBStore')
+const MetaDataStore = require('./MetaDataStore')
 
 /**
 * Restore Files plugin — restores selected files and resumes uploads
@@ -17,16 +18,22 @@ module.exports = class RestoreFiles extends Plugin {
     this.title = 'Restore Files'
 
     const defaultOptions = {
+      expires: 24 * 60 * 60 * 1000, // 24 hours
       serviceWorker: false
     }
 
     this.opts = Object.assign({}, defaultOptions, opts)
 
+    this.MetaDataStore = new MetaDataStore({
+      expires: this.opts.expires,
+      storeName: core.getID()
+    })
     this.ServiceWorkerStore = null
     if (this.opts.serviceWorker) {
-      this.ServiceWorkerStore = new ServiceWorkerStore(core, { storeName: core.getID() })
+      this.ServiceWorkerStore = new ServiceWorkerStore({ storeName: core.getID() })
     }
-    this.IndexedDBStore = new IndexedDBStore(core, Object.assign({},
+    this.IndexedDBStore = new IndexedDBStore(Object.assign(
+      { expires: this.opts.expires },
       opts.indexedDB || {},
       { storeName: core.getID() }))
 
@@ -38,11 +45,11 @@ module.exports = class RestoreFiles extends Plugin {
   }
 
   loadFilesStateFromLocalStorage () {
-    const savedState = localStorage.getItem(`uppyState:${this.core.opts.id}`)
+    const savedState = this.MetaDataStore.load()
 
     if (savedState) {
       this.core.log('Recovered some state from Local Storage')
-      this.core.setState(JSON.parse(savedState))
+      this.core.setState(savedState)
     }
   }
 
@@ -92,11 +99,10 @@ module.exports = class RestoreFiles extends Plugin {
       this.getUploadingFiles()
     )
 
-    const state = JSON.stringify({
+    this.MetaDataStore.save({
       currentUploads: this.core.state.currentUploads,
       files: filesToSave
     })
-    localStorage.setItem(`uppyState:${this.core.opts.id}`, state)
   }
 
   loadFileBlobsFromServiceWorker () {

website/src/docs/golder-retriever.md

@@ -23,20 +23,11 @@ require('uppy/lib/GoldenRetriever/ServiceWorker.js')
 uppy.use(RestoreFiles, {serviceWorker: true})
 uppy.run()
 
-const isServiceWorkerControllerReady = new Promise(resolve => {
-  if (navigator.serviceWorker.controller) return resolve()
-  navigator.serviceWorker.addEventListener('controllerchange', e => resolve())
-})
-
 if ('serviceWorker' in navigator) {
   navigator.serviceWorker
     .register('/sw.js') // path to your bundled service worker with Golden Retriever service worker
     .then((registration) => {
       console.log('ServiceWorker registration successful with scope: ', registration.scope)
-      return isServiceWorkerControllerReady
-    })
-    .then(() => {
-      uppy.emit('core:file-sw-ready')
     })
     .catch((error) => {
       console.log('Registration failed with ' + error)