diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f6ed85cd1..849d52df5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,7 @@ To be released: 2017-10-27 - [ ] plugins: add tabindex="0" to buttons and tabs? - [ ] goldenretriever: add “ghost” files (@arturi) - [ ] xhrupload: set a timeout in the onprogress event handler to detect stale network +- [ ] goldenretriever: add expiration to ServiceWorkerStore (@goto-bus-stop) # next @@ -120,7 +121,7 @@ Theme: React and Retry - [x] core: fix `replaceTargetContent` and add tests for `Plugin` (#354 / @gavboulton) - [x] goldenretriever: Omit completed uploads from saved file state—previously, when an upload was finished and the user refreshed the page, all the finished files would still be there because we saved the entire list of files. Changed this to only store files that are part of an in-progress upload, or that have yet to be uploaded (#358, #324 / @goto-bus-stop) - [x] goldenretriever: Remove files from cache when upload finished—this uses the deleteBlobs function when core:success fires (#358, #324 / @goto-bus-stop) -- [ ] goldenretriever: add a timestamp to cached blobs, and to delete old blobs on boot (#358, #324 / @goto-bus-stop) +- [x] goldenretriever: add a timestamp to cached blobs, and to delete old blobs on boot (#358, #324 / @goto-bus-stop) - [x] s3: have some way to configure content-disposition for uploads, see #243 (@goto-bus-stop) - [x] core: move `setPluginState` and add `getPluginState` to `Plugin` class (#363 / @goto-bus-stop) diff --git a/examples/bundled-example/main.js b/examples/bundled-example/main.js index 7b1b1ac115..abd30f5aeb 100644 --- a/examples/bundled-example/main.js +++ b/examples/bundled-example/main.js @@ -76,20 +76,11 @@ uppy.on('core:success', (fileList) => { console.log(fileList) }) -const isServiceWorkerControllerReady = new Promise(resolve => { - if (navigator.serviceWorker.controller) return resolve() - navigator.serviceWorker.addEventListener('controllerchange', e => resolve()) -}) - if ('serviceWorker' in navigator) { navigator.serviceWorker .register('/sw.js') .then((registration) => { console.log('ServiceWorker registration successful with scope: ', registration.scope) - return isServiceWorkerControllerReady - }) - .then(() => { - uppy.emit('core:file-sw-ready') }) .catch((error) => { console.log('Registration failed with ' + error) diff --git a/src/plugins/RestoreFiles/IndexedDBStore.js b/src/plugins/RestoreFiles/IndexedDBStore.js index 761d25413d..5efe1974a7 100644 --- a/src/plugins/RestoreFiles/IndexedDBStore.js +++ b/src/plugins/RestoreFiles/IndexedDBStore.js @@ -1,19 +1,49 @@ +const prettyBytes = require('prettier-bytes') const indexedDB = window.indexedDB || window.webkitIndexedDB || window.mozIndexedDB || window.OIndexedDB || window.msIndexedDB const isSupported = !!indexedDB const DB_NAME = 'uppy-blobs' const STORE_NAME = 'files' // maybe have a thumbnail store in the future -const DB_VERSION = 2 +const DEFAULT_EXPIRY = 24 * 60 * 60 * 1000 // 24 hours +const DB_VERSION = 3 + +// Set default `expires` dates on existing stored blobs. +function migrateExpiration (store) { + const request = store.openCursor() + request.onsuccess = (event) => { + const cursor = event.target.result + if (!cursor) { + return + } + const entry = cursor.value + entry.expires = Date.now() + DEFAULT_EXPIRY + cursor.update(entry) + } +} function connect (dbName) { const request = indexedDB.open(dbName, DB_VERSION) return new Promise((resolve, reject) => { request.onupgradeneeded = (event) => { const db = event.target.result - const store = db.createObjectStore(STORE_NAME, { keyPath: 'id' }) - store.createIndex('store', 'store', { unique: false }) - store.transaction.oncomplete = () => { + const transaction = event.currentTarget.transaction + + if (event.oldVersion < 2) { + // Added in v2: DB structure changed to a single shared object store + const store = db.createObjectStore(STORE_NAME, { keyPath: 'id' }) + store.createIndex('store', 'store', { unique: false }) + } + + if (event.oldVersion < 3) { + // Added in v3 + const store = transaction.objectStore(STORE_NAME) + store.createIndex('expires', 'expires', { unique: false }) + + migrateExpiration(store) + } + + transaction.oncomplete = () => { resolve(db) } } @@ -33,17 +63,30 @@ function waitForRequest (request) { }) } +let cleanedUp = false class IndexedDBStore { - constructor (core, opts) { + constructor (opts) { this.opts = Object.assign({ dbName: DB_NAME, storeName: 'default', + expires: DEFAULT_EXPIRY, // 24 hours maxFileSize: 10 * 1024 * 1024, // 10 MB maxTotalSize: 300 * 1024 * 1024 // 300 MB }, opts) this.name = this.opts.storeName - this.ready = connect(this.opts.dbName) + + const createConnection = () => { + return connect(this.opts.dbName) + } + + if (!cleanedUp) { + cleanedUp = true + this.ready = IndexedDBStore.cleanup() + .then(createConnection, createConnection) + } else { + this.ready = createConnection + } } key (fileID) { @@ -131,6 +174,7 @@ class IndexedDBStore { id: this.key(file.id), fileID: file.id, store: this.name, + expires: Date.now() + this.opts.expires, data: file.data }) return waitForRequest(request) @@ -148,6 +192,38 @@ class IndexedDBStore { return waitForRequest(request) }) } + + /** + * Delete all stored blobs that have an expiry date that is before Date.now(). + * This is a static method because it deletes expired blobs from _all_ Uppy instances. + */ + static cleanup () { + return connect(DB_NAME).then((db) => { + const transaction = db.transaction([STORE_NAME], 'readwrite') + const store = transaction.objectStore(STORE_NAME) + const request = store.index('expires') + .openCursor(IDBKeyRange.upperBound(Date.now())) + return new Promise((resolve, reject) => { + request.onsuccess = (event) => { + const cursor = event.target.result + if (cursor) { + const entry = cursor.value + console.log( + '[IndexedDBStore] Deleting record', entry.fileID, + 'of size', prettyBytes(entry.data.size), + '- expired on', new Date(entry.expires)) + cursor.delete() // Ignoring return value … it's not terrible if this goes wrong. + cursor.continue() + } else { + resolve(db) + } + } + request.onerror = reject + }) + }).then((db) => { + db.close() + }) + } } IndexedDBStore.isSupported = isSupported diff --git a/src/plugins/RestoreFiles/MetaDataStore.js b/src/plugins/RestoreFiles/MetaDataStore.js new file mode 100644 index 0000000000..bc973b3ff2 --- /dev/null +++ b/src/plugins/RestoreFiles/MetaDataStore.js @@ -0,0 +1,85 @@ +/** + * Get uppy instance IDs for which state is stored. + */ +function findUppyInstances () { + const instances = [] + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i) + if (/^uppyState:/.test(key)) { + instances.push(key.slice('uppyState:'.length)) + } + } + return instances +} + +/** + * Try to JSON-parse a string, return null on failure. + */ +function maybeParse (str) { + try { + return JSON.parse(str) + } catch (err) { + return null + } +} + +let cleanedUp = false +module.exports = class MetaDataStore { + constructor (opts) { + this.opts = Object.assign({ + expires: 24 * 60 * 60 * 1000 // 24 hours + }, opts) + this.name = `uppyState:${opts.storeName}` + + if (!cleanedUp) { + cleanedUp = true + MetaDataStore.cleanup() + } + } + + /** + * + */ + load () { + const savedState = localStorage.getItem(this.name) + if (!savedState) return null + const data = maybeParse(savedState) + if (!data) return null + + // Upgrade pre-0.20.0 uppyState: it used to be just a flat object, + // without `expires`. + if (!data.metadata) { + this.save(data) + return data + } + + return data.metadata + } + + save (metadata) { + const expires = Date.now() + this.opts.expires + const state = JSON.stringify({ + metadata, + expires + }) + localStorage.setItem(this.name, state) + } + + /** + * Remove all expired state. + */ + static cleanup () { + const instanceIDs = findUppyInstances() + const now = Date.now() + instanceIDs.forEach((id) => { + const data = localStorage.getItem(`uppyState:${id}`) + if (!data) return null + const obj = maybeParse(data) + if (!obj) return null + + if (obj.expires && obj.expires < now) { + localStorage.removeItem(`uppyState:${id}`) + } + }) + } +} diff --git a/src/plugins/RestoreFiles/ServiceWorkerStore.js b/src/plugins/RestoreFiles/ServiceWorkerStore.js index 279af662fd..15bc704420 100644 --- a/src/plugins/RestoreFiles/ServiceWorkerStore.js +++ b/src/plugins/RestoreFiles/ServiceWorkerStore.js @@ -1,13 +1,23 @@ const isSupported = 'serviceWorker' in navigator -class ServiceWorkerStore { - constructor (core, opts) { - this.core = core - this.ready = new Promise((resolve, reject) => { - this.core.on('core:file-sw-ready', () => { +function waitForServiceWorker () { + return new Promise((resolve, reject) => { + if (!('serviceWorker' in navigator)) { + reject(new Error('Unsupported')) + } else if (navigator.serviceWorker.controller) { + // A serviceWorker is already registered and active. + resolve() + } else { + navigator.serviceWorker.addEventListener('controllerchange', () => { resolve() }) - }) + } + }) +} + +class ServiceWorkerStore { + constructor (opts) { + this.ready = waitForServiceWorker() this.name = opts.storeName } diff --git a/src/plugins/RestoreFiles/cleanup.js b/src/plugins/RestoreFiles/cleanup.js new file mode 100644 index 0000000000..e54e7919c0 --- /dev/null +++ b/src/plugins/RestoreFiles/cleanup.js @@ -0,0 +1,10 @@ +const IndexedDBStore = require('./IndexedDBStore') +const MetaDataStore = require('./MetaDataStore') + +/** + * Clean old blobs without needing to import all of Uppy. + */ +module.exports = function cleanup () { + MetaDataStore.cleanup() + IndexedDBStore.cleanup() +} diff --git a/src/plugins/RestoreFiles/index.js b/src/plugins/RestoreFiles/index.js index 3afc4fd8ab..cda55dbfcf 100644 --- a/src/plugins/RestoreFiles/index.js +++ b/src/plugins/RestoreFiles/index.js @@ -1,6 +1,7 @@ const Plugin = require('../Plugin') const ServiceWorkerStore = require('./ServiceWorkerStore') const IndexedDBStore = require('./IndexedDBStore') +const MetaDataStore = require('./MetaDataStore') /** * Restore Files plugin — restores selected files and resumes uploads @@ -17,16 +18,22 @@ module.exports = class RestoreFiles extends Plugin { this.title = 'Restore Files' const defaultOptions = { + expires: 24 * 60 * 60 * 1000, // 24 hours serviceWorker: false } this.opts = Object.assign({}, defaultOptions, opts) + this.MetaDataStore = new MetaDataStore({ + expires: this.opts.expires, + storeName: core.getID() + }) this.ServiceWorkerStore = null if (this.opts.serviceWorker) { - this.ServiceWorkerStore = new ServiceWorkerStore(core, { storeName: core.getID() }) + this.ServiceWorkerStore = new ServiceWorkerStore({ storeName: core.getID() }) } - this.IndexedDBStore = new IndexedDBStore(core, Object.assign({}, + this.IndexedDBStore = new IndexedDBStore(Object.assign( + { expires: this.opts.expires }, opts.indexedDB || {}, { storeName: core.getID() })) @@ -38,11 +45,11 @@ module.exports = class RestoreFiles extends Plugin { } loadFilesStateFromLocalStorage () { - const savedState = localStorage.getItem(`uppyState:${this.core.opts.id}`) + const savedState = this.MetaDataStore.load() if (savedState) { this.core.log('Recovered some state from Local Storage') - this.core.setState(JSON.parse(savedState)) + this.core.setState(savedState) } } @@ -92,11 +99,10 @@ module.exports = class RestoreFiles extends Plugin { this.getUploadingFiles() ) - const state = JSON.stringify({ + this.MetaDataStore.save({ currentUploads: this.core.state.currentUploads, files: filesToSave }) - localStorage.setItem(`uppyState:${this.core.opts.id}`, state) } loadFileBlobsFromServiceWorker () { diff --git a/website/src/docs/golder-retriever.md b/website/src/docs/golder-retriever.md index 525262ba81..ebde73b2d2 100644 --- a/website/src/docs/golder-retriever.md +++ b/website/src/docs/golder-retriever.md @@ -23,20 +23,11 @@ require('uppy/lib/GoldenRetriever/ServiceWorker.js') uppy.use(RestoreFiles, {serviceWorker: true}) uppy.run() -const isServiceWorkerControllerReady = new Promise(resolve => { - if (navigator.serviceWorker.controller) return resolve() - navigator.serviceWorker.addEventListener('controllerchange', e => resolve()) -}) - if ('serviceWorker' in navigator) { navigator.serviceWorker .register('/sw.js') // path to your bundled service worker with Golden Retriever service worker .then((registration) => { console.log('ServiceWorker registration successful with scope: ', registration.scope) - return isServiceWorkerControllerReady - }) - .then(() => { - uppy.emit('core:file-sw-ready') }) .catch((error) => { console.log('Registration failed with ' + error)