Merge pull request #509 from ipfs-shipyard/fix/multiple-file-failure

This PR:

- provides a fix for a bug described in https://github.com/ipfs-shipyard/ipfs-companion via workaround from ipfs/kubo#5168 (comment)
   - **TL;DR** Browser extension is able to detect requests to `/api/v0/add` and set `Connection: close`
- moves core upload logic from background page to `quick-upload` page

add-on/src/lib/ipfs-companion.js

@@ -141,6 +141,19 @@ module.exports = async function init () {
       return
     }
     if (request.url.startsWith(state.apiURLString)) {
+      // There is a bug in go-ipfs related to keep-alive connections
+      // that results in partial response for ipfs.files.add
+      // mangled by error "http: invalid Read on closed Body"
+      // More info: https://github.com/ipfs/go-ipfs/issues/5168
+      if (request.url.includes('api/v0/add')) {
+        for (let header of request.requestHeaders) {
+          if (header.name === 'Connection') {
+            console.log('[ipfs-companion] Executing "Connection: close" workaround for https://github.com/ipfs/go-ipfs/issues/5168')
+            header.value = 'close'
+            break
+          }
+        }
+      }
       // For some reason js-ipfs-api sent requests with "Origin: null" under Chrome
       // which produced '403 - Forbidden' error.
       // This workaround removes bogus header from API requests
@@ -236,6 +249,7 @@ module.exports = async function init () {
   // ===================================================================
 
   function preloadAtPublicGateway (path) {
+    if (!state.preloadAtPublicGateway) return
     // asynchronous HTTP HEAD request preloads triggers content without downloading it
     return new Promise((resolve, reject) => {
       const http = new XMLHttpRequest()
@@ -299,7 +313,7 @@ module.exports = async function init () {
       return
     }
 
-    return uploadResultHandler(result)
+    return uploadResultHandler({result, openRootInNewTab: true})
   }
 
   // TODO: feature detect and push to client type specific modules.
@@ -314,21 +328,18 @@ module.exports = async function init () {
     }
   }
 
-  async function uploadResultHandler (result) {
+  async function uploadResultHandler ({result, openRootInNewTab = false}) {
     for (let file of result) {
       if (file && file.hash) {
         const {path, url} = getIpfsPathAndNativeAddress(file.hash)
+        preloadAtPublicGateway(path)
+        console.info('[ipfs-companion] successfully stored', file)
         // open the wrapping directory (or the CID if wrapping was disabled)
-        if (result.length === 1 || file.path === '' || file.path === file.hash) {
+        if (openRootInNewTab && (result.length === 1 || file.path === '' || file.path === file.hash)) {
           await browser.tabs.create({
             'url': url
           })
         }
-        // preload every item
-        if (state.preloadAtPublicGateway) {
-          preloadAtPublicGateway(path)
-        }
-        console.info('[ipfs-companion] successfully stored', file)
       }
     }
     return result
@@ -636,20 +647,12 @@ module.exports = async function init () {
       return ipfs
     },
 
-    async ipfsAddAndShow (data, options) {
-      options = options || {}
-      let result
-      try {
-        result = await api.ipfs.files.add(data, options)
-        if (options.wrapWithDirectory && result.length !== data.length + 1) {
-          throw new Error(`ipfs.files.add result should include an entry for every uploaded file plus additional one for a wrapping directory (${data.length + 1} in total), but found only ${result.length} entries`)
-        }
-      } catch (err) {
-        console.error('Failed to IPFS add', err)
-        notify('notify_uploadErrorTitle', 'notify_inlineErrorMsg', `${err.message}`)
-        throw err
-      }
-      return uploadResultHandler(result)
+    get notify () {
+      return notify
+    },
+
+    get uploadResultHandler () {
+      return uploadResultHandler
     },
 
     destroy () {

add-on/src/popup/quick-upload.js

@@ -26,6 +26,20 @@ function file2buffer (file) {
   })
 } */
 
+function files2streams (files) {
+  const streams = []
+  let totalSize = 0
+  for (let file of files) {
+    const fileStream = fileReaderPullStream(file, {chunkSize: 32 * 1024 * 1024})
+    streams.push({
+      path: file.name,
+      content: fileStream
+    })
+    totalSize += file.size
+  }
+  return { streams, totalSize }
+}
+
 function progressHandler (doneBytes, totalBytes, state, emitter) {
   state.message = browser.i18n.getMessage('quickUpload_state_uploading')
   // console.log('Upload progress:', doneBytes)
@@ -74,17 +88,7 @@ function quickUploadStore (state, emitter) {
     try {
       const { ipfsCompanion } = await browser.runtime.getBackgroundPage()
       const uploadTab = await browser.tabs.getCurrent()
-      const files = []
-      let totalSize = 0
-      for (let file of event.target.files) {
-        // const uploadContent = await file2buffer(file)
-        const uploadContent = fileReaderPullStream(file, {chunkSize: 32 * 1024 * 1024})
-        files.push({
-          path: file.name,
-          content: uploadContent
-        })
-        totalSize += file.size
-      }
+      let {streams, totalSize} = files2streams(event.target.files)
       if (!browser.runtime.id.includes('@')) {
         // we are in non-Firefox runtime (we know for a fact that Chrome puts no @ in id)
         if (state.ipfsNodeType === 'external' && totalSize >= 134217728) {
@@ -96,13 +100,29 @@ function quickUploadStore (state, emitter) {
       }
       progressHandler(0, totalSize, state, emitter)
       emitter.emit('render')
-      const wrapFlag = (state.wrapWithDirectory || files.length > 1)
-      const uploadOptions = {
+      const wrapFlag = (state.wrapWithDirectory || streams.length > 1)
+      const options = {
         progress: (len) => progressHandler(len, totalSize, state, emitter),
         wrapWithDirectory: wrapFlag,
         pin: state.pinUpload
       }
-      const result = await ipfsCompanion.ipfsAddAndShow(files, uploadOptions)
+      let result
+      try {
+        result = await ipfsCompanion.ipfs.files.add(streams, options)
+        // This is just an additional safety check, as in past combination
+        // of specific go-ipfs/js-ipfs-api versions
+        // produced silent errors in form of partial responses:
+        // https://github.com/ipfs-shipyard/ipfs-companion/issues/480
+        const partialResponse = result.length !== streams.length + (options.wrapWithDirectory ? 1 : 0)
+        if (partialResponse) {
+          throw new Error('Result of ipfs.files.add call is missing entries. This may be due to a bug in HTTP API similar to https://github.com/ipfs/go-ipfs/issues/5168')
+        }
+        await ipfsCompanion.uploadResultHandler({result, openRootInNewTab: true})
+      } catch (err) {
+        console.error('Failed to IPFS add', err)
+        ipfsCompanion.notify('notify_uploadErrorTitle', 'notify_inlineErrorMsg', `${err.message}`)
+        throw err
+      }
       emitter.emit('render')
       console.log('Upload result', result)
       // close upload tab as it will be replaced with a new tab with uploaded content