From f1eb595e417abe88cff3f8c147df14618bca3f58 Mon Sep 17 00:00:00 2001 From: Pedro Teixeira Date: Thu, 23 Mar 2017 22:12:08 +0000 Subject: [PATCH] fix: options to the HTTP API --- src/cli/commands/daemon.js | 15 +++++- src/cli/commands/files/add.js | 15 +++++- src/core/components/files.js | 6 ++- src/core/index.js | 3 ++ src/http-api/index.js | 3 +- test/core/files-sharding.spec.js | 89 ++++++++++++++++++++++++++++++++ 6 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 test/core/files-sharding.spec.js diff --git a/src/cli/commands/daemon.js b/src/cli/commands/daemon.js index 076a6d9fbe..751b0b3611 100644 --- a/src/cli/commands/daemon.js +++ b/src/cli/commands/daemon.js @@ -10,11 +10,22 @@ module.exports = { describe: 'Start a long-running daemon process', - handler () { + builder: { + 'enable-sharding-experiment': { + type: 'boolean', + defaultt: false + }, + 'enable-pubsub-experiment': { + type: 'booleam', + default: false + } + }, + + handler (argv) { console.log('Initializing daemon...') const repoPath = utils.getRepoPath() - httpAPI = new HttpAPI(repoPath) + httpAPI = new HttpAPI(process.env.IPFS_PATH, argv) httpAPI.start((err) => { if (err && err.code === 'ENOENT') { diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index e133056c16..a39e7d1848 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -56,6 +56,14 @@ module.exports = { alias: 'w', type: 'boolean', default: false + }, + 'enable-sharding-experiment': { + type: 'boolean', + defaultt: false + }, + 'shard-split-threshold': { + type: 'integer', + default: 1000 } }, @@ -63,7 +71,12 @@ module.exports = { const inPath = checkPath(argv.file, argv.recursive) const index = inPath.lastIndexOf('/') + 1 const options = { - strategy: argv.trickle ? 'trickle' : 'balanced' + strategy: argv.trickle ? 'trickle' : 'balanced', + shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity + } + + if (argv.enableShardingExperiment && utils.isDaemonOn()) { + throw new Error('Error: Enabling the sharding experiment should be done on the daemon') } const ipfs = argv.ipfs diff --git a/src/core/components/files.js b/src/core/components/files.js index b84cff3fef..a5e28e7f7e 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -18,10 +18,14 @@ const Duplex = require('stream').Duplex module.exports = function files (self) { const createAddPullStream = (options) => { + const opts = Object.assign({}, { + shardSplitThreshold: self._options.EXPERIMENTAL.sharding ? 1000 : Infinity + }, options) + return pull( pull.map(normalizeContent), pull.flatten(), - importer(self._ipldResolver, options), + importer(self._ipldResolver, opts), pull.asyncMap(prepareFile.bind(null, self)) ) } diff --git a/src/core/index.js b/src/core/index.js index 042d7731e5..f0b7c0c815 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -97,6 +97,9 @@ class IPFS extends EventEmitter { if (this._options.EXPERIMENTAL.pubsub) { this.log('EXPERIMENTAL pubsub is enabled') } + if (this._options.EXPERIMENTAL.sharding) { + this.log('EXPERIMENTAL sharding is enabled') + } this.state = require('./state')(this) boot(this) diff --git a/src/http-api/index.js b/src/http-api/index.js index 3fb36e73b2..0c93b889e2 100644 --- a/src/http-api/index.js +++ b/src/http-api/index.js @@ -40,7 +40,8 @@ function HttpApi (repo, config) { start: true, config: config, EXPERIMENTAL: { - pubsub: true + pubsub: true, + sharding: config && config.enableShardingExperiment } }) } catch (err) { diff --git a/test/core/files-sharding.spec.js b/test/core/files-sharding.spec.js new file mode 100644 index 0000000000..77e8d2d158 --- /dev/null +++ b/test/core/files-sharding.spec.js @@ -0,0 +1,89 @@ +'use strict' + +const expect = require('chai').expect +const pull = require('pull-stream') + +const IPFS = require('../../src/core') +const createTempRepo = require('../utils/create-repo-node.js') + +describe('files dir', () => { + + const files = [] + for(let i = 0; i < 1005; i++) { + files.push({ + path: 'test-folder/' + i, + content: new Buffer('some content ' + i) + }) + } + + describe('without sharding', () => { + let ipfs + + before((done) => { + ipfs = new IPFS({ + repo: createTempRepo(), + config: { + Bootstrap: [] + } + }) + ipfs.once('start', done) + }) + + after((done) => { + ipfs.stop(done) + }) + + it('should be able to add dir without sharding', (done) => { + pull( + pull.values(files), + ipfs.files.createAddPullStream(), + pull.collect((err, results) => { + const last = results[results.length - 1] + expect(last.path).to.be.eql('test-folder') + expect(last.hash).to.be.eql('QmWWM8ZV6GPhqJ46WtKcUaBPNHN5yQaFsKDSQ1RE73w94Q') + done() + }) + ) + + after((done) => { + ipfs.stop(() => done()) // ignore stop errors + }) + + }).timeout(4000) + }) + + describe('with sharding', () => { + let ipfs + + before((done) => { + ipfs = new IPFS({ + repo: createTempRepo(), + config: { + Bootstrap: [] + }, + EXPERIMENTAL: { + sharding: true + } + }) + ipfs.once('start', done) + }) + + after((done) => { + ipfs.stop(() => done()) // ignore stop errors + }) + + it('should be able to add dir with sharding', (done) => { + pull( + pull.values(files), + ipfs.files.createAddPullStream(), + pull.collect((err, results) => { + const last = results[results.length - 1] + expect(last.path).to.be.eql('test-folder') + expect(last.hash).to.be.eql('QmZjYC1kWrLmiRYbEmGSo2PEpMixzT2k2xoCKSBzt8KDcy') + done() + }) + ) + + }).timeout(4000) + }) +})