feat: Allows for byte offsets when using ipfs.files.cat and friends t…

…o request slices of files feat: Allows for byte offsets when using ipfs.files.cat and friends to request slices of files
ipfs · May 1, 2018 · fff0aa1 · fff0aa1
1 parent 6248cac
commit fff0aa1
Show file tree

Hide file tree

Showing 12 changed files with 446 additions and 18 deletions.
diff --git a/examples/README.md b/examples/README.md
@@ -22,6 +22,7 @@ Let us know if you find any issue or if you want to contribute and add a new tut
 - [js-ipfs in electron](./run-in-electron)
 - [Using streams to add a directory of files to ipfs](./browser-add-readable-stream)
 - [Customizing the ipfs repository](./custom-ipfs-repo)
+- - [Streaming video from ipfs to the browser using `ReadableStream`s](./browser-readablestream)
 
 ## Understanding the IPFS Stack
 

diff --git a/examples/browser-readablestream/README.md b/examples/browser-readablestream/README.md
@@ -0,0 +1,16 @@
+# Streaming video from IPFS using ReadableStreams
+
+We can use the execllent [`videostream`](https://www.npmjs.com/package/videostream) to stream video from IPFS to the browser.  All we need to do is return a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream)-like object that contains the requested byte ranges.
+
+Take a look at [`index.js`](./index.js) to see a working example.
+
+## Running the demo
+
+In this directory:
+
+```
+$ npm install
+$ npm start
+```
+
+Then open [http://localhost:8888](http://localhost:8888) in your browser.
diff --git a/examples/browser-readablestream/index.html b/examples/browser-readablestream/index.html
@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8"/>
+    <title><%= htmlWebpackPlugin.options.title %></title>
+    <style type="text/css">
+
+body {
+  margin: 0;
+  padding: 0;
+}
+
+#container {
+  display: flex;
+  height: 100vh;
+}
+
+pre {
+  flex-grow: 2;
+  padding: 10px;
+  height: calc(100vh - 45px);
+  overflow: auto;
+}
+
+#form-wrapper {
+  padding: 20px;
+}
+
+form {
+  padding-bottom: 10px;
+  display: flex;
+}
+
+#hash {
+  display: inline-block;
+  margin: 0 10px 10px 0;
+  font-size: 16px;
+  flex-grow: 2;
+  padding: 5px;
+}
+
+button {
+  display: inline-block;
+  font-size: 16px;
+  height: 32px;
+}
+
+video {
+  max-width: 50vw;
+}
+
+    </style>
+  </head>
+  <body>
+    <div id="container" ondrop="dropHandler(event)" ondragover="dragOverHandler(event)">
+      <div id="form-wrapper">
+          <form>
+            <input type="text" id="hash" placeholder="Hash" disabled />
+            <button id="gobutton" disabled>Go!</button>
+          </form>
+          <video id="video" controls></video>
+      </div>
+      <pre id="output" style="display: inline-block"></pre>
+    </div>
+  </body>
+</html>
diff --git a/examples/browser-readablestream/index.js b/examples/browser-readablestream/index.js
@@ -0,0 +1,75 @@
+'use strict'
+
+/* eslint-env browser */
+
+const Ipfs = require('../../')
+const videoStream = require('videostream')
+const ipfs = new Ipfs({ repo: 'ipfs-' + Math.random() })
+const {
+  dragDrop,
+  statusMessages,
+  createVideoElement,
+  log
+} = require('./utils')
+
+log('IPFS: Initialising')
+
+ipfs.on('ready', () => {
+  // Set up event listeners on the <video> element from index.html
+  const videoElement = createVideoElement()
+  const hashInput = document.getElementById('hash')
+  const goButton = document.getElementById('gobutton')
+  let stream
+
+  goButton.onclick = function (event) {
+    event.preventDefault()
+
+    log(`IPFS: Playing ${hashInput.value.trim()}`)
+
+    // Set up the video stream an attach it to our <video> element
+    videoStream({
+      createReadStream: function createReadStream (opts) {
+        const start = opts.start
+
+        // The videostream library does not always pass an end byte but when
+        // it does, it wants bytes between start & end inclusive.
+        // catReadableStream returns the bytes exclusive so increment the end
+        // byte if it's been requested
+        const end = opts.end ? start + opts.end + 1 : undefined
+
+        log(`Stream: Asked for data starting at byte ${start} and ending at byte ${end}`)
+
+        // If we've streamed before, clean up the existing stream
+        if (stream && stream.destroy) {
+          stream.destroy()
+        }
+
+        // This stream will contain the requested bytes
+        stream = ipfs.files.catReadableStream(hashInput.value.trim(), {
+          offset: start,
+          length: end && end - start
+        })
+
+        // Log error messages
+        stream.on('error', (error) => log(error))
+
+        if (start === 0) {
+          // Show the user some messages while we wait for the data stream to start
+          statusMessages(stream, log)
+        }
+
+        return stream
+      }
+    }, videoElement)
+  }
+
+  // Allow adding files to IPFS via drag and drop
+  dragDrop(ipfs, log)
+
+  log('IPFS: Ready')
+  log('IPFS: Drop an .mp4 file into this window to add a file')
+  log('IPFS: Then press the "Go!" button to start playing a video')
+
+  hashInput.disabled = false
+  goButton.disabled = false
+})
diff --git a/examples/browser-readablestream/package.json b/examples/browser-readablestream/package.json
@@ -0,0 +1,22 @@
+{
+  "name": "browser-videostream",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "webpack",
+    "start": "npm run build && http-server dist -a 127.0.0.1 -p 8888"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "html-webpack-plugin": "^2.30.1",
+    "http-server": "^0.11.1",
+    "uglifyjs-webpack-plugin": "^1.2.0",
+    "webpack": "^3.11.0"
+  },
+  "dependencies": {
+    "videostream": "^2.4.2"
+  }
+}
diff --git a/examples/browser-readablestream/utils.js b/examples/browser-readablestream/utils.js
@@ -0,0 +1,145 @@
+'use strict'
+
+const log = (line) => {
+  const output = document.getElementById('output')
+  let message
+
+  if (line.message) {
+    message = `Error: ${line.message.toString()}`
+  } else {
+    message = line
+  }
+
+  if (message) {
+    const node = document.createTextNode(`${message}\r\n`)
+    output.appendChild(node)
+
+    output.scrollTop = output.offsetHeight
+
+    return node
+  }
+}
+
+const dragDrop = (ipfs) => {
+  const container = document.querySelector('#container')
+
+  container.ondragover = (event) => {
+    event.preventDefault()
+  }
+
+  container.ondrop = (event) => {
+    event.preventDefault()
+
+    Array.prototype.slice.call(event.dataTransfer.items)
+      .filter(item => item.kind === 'file')
+      .map(item => item.getAsFile())
+      .forEach(file => {
+        const progress = log(`IPFS: Adding ${file.name} 0%`)
+
+        const reader = new window.FileReader()
+        reader.onload = (event) => {
+          ipfs.files.add({
+            path: file.name,
+            content: ipfs.types.Buffer.from(event.target.result)
+          }, {
+            progress: (addedBytes) => {
+              progress.textContent = `IPFS: Adding ${file.name} ${parseInt((addedBytes / file.size) * 100)}%\r\n`
+            }
+          }, (error, added) => {
+            if (error) {
+              return log(error)
+            }
+
+            const hash = added[0].hash
+
+            log(`IPFS: Added ${hash}`)
+
+            document.querySelector('#hash').value = hash
+          })
+        }
+
+        reader.readAsArrayBuffer(file)
+      })
+
+    if (event.dataTransfer.items && event.dataTransfer.items.clear) {
+      event.dataTransfer.items.clear()
+    }
+
+    if (event.dataTransfer.clearData) {
+      event.dataTransfer.clearData()
+    }
+  }
+}
+
+module.exports.statusMessages = (stream) => {
+  let time = 0
+  const timeouts = [
+    'Stream: Still loading data from IPFS...',
+    'Stream: This can take a while depending on content availability',
+    'Stream: Hopefully not long now',
+    'Stream: *Whistles absentmindedly*',
+    'Stream: *Taps foot*',
+    'Stream: *Looks at watch*',
+    'Stream: *Stares at floor*',
+    'Stream: *Checks phone*',
+    'Stream: *Stares at ceiling*',
+    'Stream: Got anything nice planned for the weekend?'
+  ].map(message => {
+    time += 5000
+
+    return setTimeout(() => {
+      log(message)
+    }, time)
+  })
+
+  stream.once('data', () => {
+    log('Stream: Started receiving data')
+    timeouts.forEach(clearTimeout)
+  })
+  stream.once('error', () => {
+    timeouts.forEach(clearTimeout)
+  })
+}
+
+const createVideoElement = () => {
+  const videoElement = document.getElementById('video')
+  videoElement.addEventListener('loadedmetadata', () => {
+    videoElement.play()
+      .catch(log)
+  })
+
+  const events = [
+    'playing',
+    'waiting',
+    'seeking',
+    'seeked',
+    'ended',
+    'loadedmetadata',
+    'loadeddata',
+    'canplay',
+    'canplaythrough',
+    'durationchange',
+    'play',
+    'pause',
+    'suspend',
+    'emptied',
+    'stalled',
+    'error',
+    'abort'
+  ]
+  events.forEach(event => {
+    videoElement.addEventListener(event, () => {
+      log(`Video: ${event}`)
+    })
+  })
+
+  videoElement.addEventListener('error', () => {
+    log(videoElement.error)
+  })
+
+  return videoElement
+}
+
+module.exports.log = log
+module.exports.dragDrop = dragDrop
+module.exports.createVideoElement = createVideoElement
diff --git a/examples/browser-readablestream/webpack.config.js b/examples/browser-readablestream/webpack.config.js
@@ -0,0 +1,29 @@
+'use strict'
+
+const path = require('path')
+const UglifyJsPlugin = require('uglifyjs-webpack-plugin')
+const HtmlWebpackPlugin = require('html-webpack-plugin')
+
+module.exports = {
+  devtool: 'source-map',
+  entry: [
+    './index.js'
+  ],
+  plugins: [
+    new UglifyJsPlugin({
+      sourceMap: true,
+      uglifyOptions: {
+        mangle: false,
+        compress: false
+      }
+    }),
+    new HtmlWebpackPlugin({
+      title: 'IPFS Videostream example',
+      template: 'index.html'
+    })
+  ],
+  output: {
+    path: path.join(__dirname, 'dist'),
+    filename: 'bundle.js'
+  }
+}
diff --git a/package.json b/package.json
@@ -73,7 +73,7 @@
     "expose-loader": "^0.7.5",
     "form-data": "^2.3.2",
     "hat": "0.0.3",
-    "interface-ipfs-core": "^0.64.2",
+    "interface-ipfs-core": "~0.64.2",
     "ipfsd-ctl": "^0.32.1",
     "lodash": "^4.17.10",
     "mocha": "^5.1.1",
@@ -118,6 +118,7 @@
     "ipfs-unixfs-engine": "~0.29.0",
     "ipld": "^0.17.0",
     "is-ipfs": "^0.3.2",
+    "ipld-dag-pb": "~0.14.3",
     "is-pull-stream": "0.0.0",
     "is-stream": "^1.1.0",
     "joi": "^13.2.0",
@@ -138,6 +139,7 @@
     "libp2p-websockets": "~0.12.0",
     "lodash.flatmap": "^4.5.0",
     "lodash.get": "^4.4.2",
+    "lodash.set": "^4.3.2",
     "lodash.sortby": "^4.7.0",
     "lodash.values": "^4.3.0",
     "mafmt": "^6.0.0",