From c203ede8552c6025bbb80a36fa51b6354ba9d4b4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 4 Dec 2018 15:06:05 +0000 Subject: [PATCH 1/4] feat: add files.ls streaming methods N.b will not actually do any streaming until https://github.com/ipfs/go-ipfs/pull/5611 lands --- src/files-mfs/index.js | 2 ++ src/files-mfs/ls-pull-stream.js | 21 +++++++++++++++++++++ src/files-mfs/ls-readable-stream.js | 10 ++++++++++ 3 files changed, 33 insertions(+) create mode 100644 src/files-mfs/ls-pull-stream.js create mode 100644 src/files-mfs/ls-readable-stream.js diff --git a/src/files-mfs/index.js b/src/files-mfs/index.js index d4c6a8e7f..d35e82ea7 100644 --- a/src/files-mfs/index.js +++ b/src/files-mfs/index.js @@ -12,6 +12,8 @@ module.exports = (arg) => { stat: require('./stat')(send), rm: require('./rm')(send), ls: require('./ls')(send), + lsReadableStream: require('./ls-readable-stream')(send), + lsPullStream: require('./ls-pull-stream')(send), read: require('./read')(send), readReadableStream: require('./read-readable-stream')(send), readPullStream: require('./read-pull-stream')(send), diff --git a/src/files-mfs/ls-pull-stream.js b/src/files-mfs/ls-pull-stream.js new file mode 100644 index 000000000..41b46b756 --- /dev/null +++ b/src/files-mfs/ls-pull-stream.js @@ -0,0 +1,21 @@ +'use strict' + +const ls = require('./ls') +const defer = require('pull-defer') +const values = require('pull-stream/sources/values') + +module.exports = (send) => { + return (args, opts) => { + const deferred = defer.source() + + ls(send)(args, opts, (err, entries) => { + if (err) { + return deferred.abort(err) + } + + return deferred.resolve(values(entries)) + }) + + return deferred + } +} diff --git a/src/files-mfs/ls-readable-stream.js b/src/files-mfs/ls-readable-stream.js new file mode 100644 index 000000000..2db339da7 --- /dev/null +++ b/src/files-mfs/ls-readable-stream.js @@ -0,0 +1,10 @@ +'use strict' + +const lsPullStream = require('./ls-pull-stream') +const toStream = require('pull-stream-to-stream') + +module.exports = (send) => { + return (args, opts) => { + return toStream.source(lsPullStream(send)(args, opts)) + } +} From f0fd3ed46143218ed320a2ff8b373ccca25a0816 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 4 Dec 2018 17:30:18 +0000 Subject: [PATCH 2/4] feat: streams files.ls responses from js-ipfs --- src/files-mfs/ls-pull-stream.js | 17 +++--------- src/files-mfs/ls-readable-stream.js | 41 ++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/files-mfs/ls-pull-stream.js b/src/files-mfs/ls-pull-stream.js index 41b46b756..111eba7ed 100644 --- a/src/files-mfs/ls-pull-stream.js +++ b/src/files-mfs/ls-pull-stream.js @@ -1,21 +1,12 @@ 'use strict' -const ls = require('./ls') -const defer = require('pull-defer') -const values = require('pull-stream/sources/values') +const toPull = require('stream-to-pull-stream') +const lsReadableStream = require('./ls-readable-stream') module.exports = (send) => { return (args, opts) => { - const deferred = defer.source() + opts = opts || {} - ls(send)(args, opts, (err, entries) => { - if (err) { - return deferred.abort(err) - } - - return deferred.resolve(values(entries)) - }) - - return deferred + return toPull.source(lsReadableStream(send)(args, opts)) } } diff --git a/src/files-mfs/ls-readable-stream.js b/src/files-mfs/ls-readable-stream.js index 2db339da7..9b0b3e567 100644 --- a/src/files-mfs/ls-readable-stream.js +++ b/src/files-mfs/ls-readable-stream.js @@ -1,10 +1,45 @@ 'use strict' -const lsPullStream = require('./ls-pull-stream') -const toStream = require('pull-stream-to-stream') +const { + Transform +} = require('stream') +const pump = require('pump') +const ndjson = require('ndjson') module.exports = (send) => { return (args, opts) => { - return toStream.source(lsPullStream(send)(args, opts)) + opts = opts || {} + + const transform = new Transform({ + objectMode: true, + + transform (entry, encoding, callback) { + callback(null, { + name: entry.Name, + type: entry.Type, + size: entry.Size, + hash: entry.Hash + }) + } + }) + + send({ + path: 'files/ls', + args: args, + qs: { + ...opts, + stream: true + } + }, (err, stream) => { + if (err) { + return transform.destroy(err) + } + + const outputStream = ndjson.parse() + + pump(stream, outputStream, transform) + }) + + return transform } } From beb4c4c0557e14250e6d05d5ce364da733dcfb79 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 5 Dec 2018 10:02:34 +0000 Subject: [PATCH 3/4] chore: update interface-ipfs-core dependency License: MIT Signed-off-by: Alan Shaw --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f9590cb61..a8b375ab4 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,7 @@ "eslint-plugin-react": "^7.11.1", "go-ipfs-dep": "~0.4.18", "gulp": "^3.9.1", - "interface-ipfs-core": "~0.88.0", + "interface-ipfs-core": "~0.90.0", "ipfsd-ctl": "~0.40.0", "nock": "^10.0.2", "pull-stream": "^3.6.9", From a17d89a85bf48f53d6b84811d127692a7a348148 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 5 Dec 2018 12:03:48 +0000 Subject: [PATCH 4/4] fix: simulate streaming on go-ipfs --- src/files-mfs/ls-readable-stream.js | 44 +++++++++++++++++++++-------- test/interface.spec.js | 9 +++++- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/src/files-mfs/ls-readable-stream.js b/src/files-mfs/ls-readable-stream.js index 9b0b3e567..b2107034d 100644 --- a/src/files-mfs/ls-readable-stream.js +++ b/src/files-mfs/ls-readable-stream.js @@ -1,10 +1,21 @@ 'use strict' const { - Transform + Transform, + PassThrough } = require('stream') const pump = require('pump') const ndjson = require('ndjson') +const isStream = require('is-stream') + +const toEntry = (entry) => { + return { + name: entry.Name, + type: entry.Type, + size: entry.Size, + hash: entry.Hash + } +} module.exports = (send) => { return (args, opts) => { @@ -14,15 +25,14 @@ module.exports = (send) => { objectMode: true, transform (entry, encoding, callback) { - callback(null, { - name: entry.Name, - type: entry.Type, - size: entry.Size, - hash: entry.Hash - }) + callback(null, toEntry(entry)) } }) + const output = new PassThrough({ + objectMode: true + }) + send({ path: 'files/ls', args: args, @@ -30,16 +40,26 @@ module.exports = (send) => { ...opts, stream: true } - }, (err, stream) => { + }, (err, res) => { if (err) { - return transform.destroy(err) + return output.destroy(err) } - const outputStream = ndjson.parse() + if (isStream(res)) { + const parse = ndjson.parse() + + pump(res, parse, transform, output) + } else { + const entries = res.Entries || [] - pump(stream, outputStream, transform) + entries.forEach((entry) => { + output.write(toEntry(entry)) + }) + + output.end() + } }) - return transform + return output } } diff --git a/test/interface.spec.js b/test/interface.spec.js index b726f8cb0..7a5413e14 100644 --- a/test/interface.spec.js +++ b/test/interface.spec.js @@ -163,7 +163,14 @@ describe('interface-ipfs-core tests', () => { ] }) - tests.filesMFS(defaultCommonFactory) + tests.filesMFS(defaultCommonFactory, { + only: [ + { + name: 'should ls directory', + reason: 'TODO not impemented in go-ipfs yet' + } + ] + }) tests.key(defaultCommonFactory, { skip: [