From 2c3db47e60ac6f17f30af6790f1f7e903bfe1ce2 Mon Sep 17 00:00:00 2001 From: kumavis Date: Thu, 8 Dec 2016 15:41:37 -0800 Subject: [PATCH 1/8] cli - block put - add CID options: format, mhtype, mhlen --- package.json | 2 +- src/cli/commands/block/put.js | 58 +++++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 33b412d2ed..723925ea11 100644 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "ipfs-unixfs-engine": "^0.14.0", "ipld-dag-cbor": "^0.8.2", "ipld-dag-pb": "^0.9.3", - "ipld-resolver": "^0.3.0", + "ipld-resolver": "^0.4.0", "isstream": "^0.1.2", "joi": "^10.0.1", "libp2p-ipfs-browser": "^0.17.0", diff --git a/src/cli/commands/block/put.js b/src/cli/commands/block/put.js index cdd47a7776..8ad303d0a4 100644 --- a/src/cli/commands/block/put.js +++ b/src/cli/commands/block/put.js @@ -1,32 +1,58 @@ 'use strict' const utils = require('../../utils') -const mh = require('multihashes') const bl = require('bl') const fs = require('fs') const Block = require('ipfs-block') +const CID = require('cids') const waterfall = require('async/waterfall') const debug = require('debug') const log = debug('cli:block') log.error = debug('cli:block:error') -function addBlock (buf) { +function addBlock (buf, opts) { + let block = new Block(buf) + let mhash, cid + utils.getIPFS((err, ipfs) => { if (err) { throw err } waterfall([ - (cb) => ipfs.block.put(new Block(buf), cb), - (block, cb) => block.key(cb) - ], (err, key) => { + (cb) => generateHash(block, opts, cb), + (cb) => generateCid(mhash, block, opts, cb), + (cb) => ipfs.block.put(block, cid, cb) + ], (err) => { if (err) { throw err } - console.log(mh.toB58String(key)) + console.log(cid.toBaseEncodedString()) }) }) + + function generateHash (block, opts, cb) { + if (opts.format === 'v0') { + block.key(done) + } else { + block.key(opts.mhtype, done) + } + function done (err, _mhash) { + if (err) return cb(err) + mhash = _mhash + cb() + } + } + + function generateCid (mhash, block, opts, cb) { + if (opts.format === 'v0') { + cid = new CID(0, 'dag-pb', mhash) + } else { + cid = new CID(1, opts.format, mhash) + } + cb() + } } module.exports = { @@ -34,11 +60,25 @@ module.exports = { describe: 'Stores input as an IPFS block', - builder: {}, + builder: { + format: { + alias: 'f', + describe: 'cid format for blocks to be created with.', + default: 'v0' + }, + mhtype: { + describe: 'multihash hash function', + default: 'sha2-256' + }, + mhlen: { + describe: 'multihash hash length', + default: undefined + } + }, handler (argv) { if (argv.data) { - return addBlock(fs.readFileSync(argv.data)) + return addBlock(fs.readFileSync(argv.data), argv) } process.stdin.pipe(bl((err, input) => { @@ -46,7 +86,7 @@ module.exports = { throw err } - addBlock(input) + addBlock(input, argv) })) } } From 55e006c19208b96d88fd564a07fd9e674d26f763 Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 9 Dec 2016 16:45:26 -0800 Subject: [PATCH 2/8] tests - add cli block put flags test --- test/cli/test-block.js | 8 ++++++++ test/test-data/eth-block | Bin 0 -> 1082 bytes 2 files changed, 8 insertions(+) create mode 100644 test/test-data/eth-block diff --git a/test/cli/test-block.js b/test/cli/test-block.js index 6beb8b3076..0649419f3f 100644 --- a/test/cli/test-block.js +++ b/test/cli/test-block.js @@ -16,6 +16,14 @@ describe('block', () => { }) }) + it('put with flags, format and mhtype', () => { + return ipfs('block put --format eth-block --mhtype keccak-256 test/test-data/eth-block').then((out) => { + expect(out).to.be.eql( + 'z43AaGF2Jd3Kt5wSYT6wbU4D6QbCXM8AtwZ6WYsVVmnAz7pfX95' + ) + }) + }) + it('get', () => { return ipfs('block get QmZjTnYw2TFhn9Nn7tjmPSoTBoY7YRkwPzwSrSbabY24Kp').then((out) => { expect(out).to.be.eql('hello world\n') diff --git a/test/test-data/eth-block b/test/test-data/eth-block new file mode 100644 index 0000000000000000000000000000000000000000..6615820ede1d0cc41ca881670626b5b198e1a654 GIT binary patch literal 1082 zcmey#V*Zm!c)^JS3yWFTzM8$;d&W{-mk$Ts`(r%*c&8N4d%>!r%D5?P0q6BD^F2S6 zGxOJby-M1(;z2%dF~7Wli=*a#MV+JjI4q}VOp^1{ncMQ^z=s3pc24{f9=PD&U_Jjv_JLYY#9c>2H0UyhV|qf}M;EBOSD|JSY{;YGPc{B;3)^61vMRYNmT?Nrrxzp`L-B zfqqVAUTKAXdcL8aiJs8{LV?v$Sydxw#KPu$;3t#pPbR4a_diBj#&6!o>ayq2X4#zI zJ})a9+b4c_{&!Qf^1QXOWt|IT&-lK$cRaRgb?esjZD+1XT^4ecp3+q!{9&QfUxzD` z-KPX}TvOh~f1c6sL1)#(VlUt3$gBkm;)K`Sbeg*>@$U}x6`KABqpoO#U8-QIx89i< z@h{* Date: Fri, 9 Dec 2016 16:46:33 -0800 Subject: [PATCH 3/8] cli - block get + info - add CID support --- package.json | 5 +++-- src/cli/commands/block/get.js | 12 ++++++------ src/cli/commands/block/stat.js | 5 ++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 723925ea11..6b0d2b43ab 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "async": "^2.1.4", "bl": "^1.1.2", "boom": "^4.2.0", + "cids": "^0.3.4", "debug": "^2.3.3", "fs-pull-blob-store": "^0.3.0", "glob": "^7.1.1", @@ -81,9 +82,9 @@ "hapi-set-header": "^1.0.2", "idb-pull-blob-store": "^0.5.1", "ipfs-api": "^12.0.0", - "ipfs-bitswap": "^0.8.1", + "ipfs-bitswap": "^0.8.2", "ipfs-block": "^0.5.0", - "ipfs-block-service": "^0.7.0", + "ipfs-block-service": "^0.7.1", "ipfs-multipart": "^0.1.0", "ipfs-repo": "^0.11.1", "ipfs-unixfs": "^0.1.8", diff --git a/src/cli/commands/block/get.js b/src/cli/commands/block/get.js index f042eb0632..ed0218f03b 100644 --- a/src/cli/commands/block/get.js +++ b/src/cli/commands/block/get.js @@ -1,7 +1,7 @@ 'use strict' const utils = require('../../utils') -const mh = require('multihashes') +const CID = require('cids') const debug = require('debug') const log = debug('cli:block') log.error = debug('cli:block:error') @@ -19,17 +19,17 @@ module.exports = { throw err } - const hash = utils.isDaemonOn() - ? argv.key - : mh.fromB58String(argv.key) + const cid = new CID(argv.key) - ipfs.block.get(hash, (err, block) => { + ipfs.block.get(cid, (err, block) => { if (err) { throw err } if (block.data) { - console.log(block.data.toString()) + // writing the buffer to stdout seems to be the only way + // to send out binary data correctly + process.stdout.write(block.data) return } diff --git a/src/cli/commands/block/stat.js b/src/cli/commands/block/stat.js index ab7887c75b..a3cefd2f70 100644 --- a/src/cli/commands/block/stat.js +++ b/src/cli/commands/block/stat.js @@ -1,6 +1,7 @@ 'use strict' const utils = require('../../utils') +const CID = require('cids') const debug = require('debug') const log = debug('cli:block') log.error = debug('cli:block:error') @@ -13,12 +14,14 @@ module.exports = { builder: {}, handler (argv) { + const cid = new CID(argv.key) + utils.getIPFS((err, ipfs) => { if (err) { throw err } - ipfs.block.stat(argv.key, (err, stats) => { + ipfs.block.stat(cid, (err, stats) => { if (err) { throw err } From 1e6fca8627116b3d0c3967cb6d2e217102899c4d Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 9 Dec 2016 19:37:36 -0800 Subject: [PATCH 4/8] tests - cli - fix eth-block example data --- test/cli/test-block.js | 2 +- test/test-data/eth-block | Bin 1082 -> 534 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli/test-block.js b/test/cli/test-block.js index 0649419f3f..9d5c754b7a 100644 --- a/test/cli/test-block.js +++ b/test/cli/test-block.js @@ -19,7 +19,7 @@ describe('block', () => { it('put with flags, format and mhtype', () => { return ipfs('block put --format eth-block --mhtype keccak-256 test/test-data/eth-block').then((out) => { expect(out).to.be.eql( - 'z43AaGF2Jd3Kt5wSYT6wbU4D6QbCXM8AtwZ6WYsVVmnAz7pfX95' + 'z43AaGF4V81rH6y9ykccadnaGV7EkvbPhFXuvrP6mpK89r8bR5T' ) }) }) diff --git a/test/test-data/eth-block b/test/test-data/eth-block index 6615820ede1d0cc41ca881670626b5b198e1a654..438f4c207e32dc0479771c5986c990128f9f5bd1 100644 GIT binary patch delta 8 PcmdnRF^y#-vltTq45I=m delta 210 zcmV;@04@KP1iAI`e>=HpSpHD>jXaT@A+ek$0W#fb z?uLQ^pMn#JfP_`LMpB?hWprpSb}=q6E-)=-V>dZuGcsf^Y-w(FcrRyfF)lQqk-;Dd Mh*w`va8$XJ$j)V8f&c&j From ec82fe5a24ea4b0fd131a75cad1c6fdfd8eb6320 Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 9 Dec 2016 20:57:15 -0800 Subject: [PATCH 5/8] tests - cli - remove newline from block get result --- test/cli/test-block.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli/test-block.js b/test/cli/test-block.js index 9d5c754b7a..b52a04e983 100644 --- a/test/cli/test-block.js +++ b/test/cli/test-block.js @@ -26,7 +26,7 @@ describe('block', () => { it('get', () => { return ipfs('block get QmZjTnYw2TFhn9Nn7tjmPSoTBoY7YRkwPzwSrSbabY24Kp').then((out) => { - expect(out).to.be.eql('hello world\n') + expect(out).to.be.eql('hello world') }) }) From 5319a4159b81b481feedd95720bdad65947d2a1f Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 9 Dec 2016 20:59:58 -0800 Subject: [PATCH 6/8] cli - block put - pass mhlength flag to multihash --- package.json | 1 + src/cli/commands/block/put.js | 33 +++++++++++++++++---------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 6b0d2b43ab..dd84c38d3e 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "mafmt": "^2.1.2", "multiaddr": "^2.1.1", "multihashes": "^0.3.0", + "multihashing-async": "^0.3.0", "path-exists": "^3.0.0", "peer-book": "^0.3.0", "peer-id": "^0.8.0", diff --git a/src/cli/commands/block/put.js b/src/cli/commands/block/put.js index 8ad303d0a4..5b86f64f93 100644 --- a/src/cli/commands/block/put.js +++ b/src/cli/commands/block/put.js @@ -5,6 +5,7 @@ const bl = require('bl') const fs = require('fs') const Block = require('ipfs-block') const CID = require('cids') +const multihashing = require('multihashing-async') const waterfall = require('async/waterfall') const debug = require('debug') const log = debug('cli:block') @@ -12,7 +13,7 @@ log.error = debug('cli:block:error') function addBlock (buf, opts) { let block = new Block(buf) - let mhash, cid + let cid utils.getIPFS((err, ipfs) => { if (err) { @@ -21,7 +22,7 @@ function addBlock (buf, opts) { waterfall([ (cb) => generateHash(block, opts, cb), - (cb) => generateCid(mhash, block, opts, cb), + (mhash, cb) => generateCid(mhash, block, opts, cb), (cb) => ipfs.block.put(block, cid, cb) ], (err) => { if (err) { @@ -33,24 +34,15 @@ function addBlock (buf, opts) { }) function generateHash (block, opts, cb) { - if (opts.format === 'v0') { - block.key(done) + if (opts.mhlen === undefined) { + multihashing(buf, opts.mhtype, cb) } else { - block.key(opts.mhtype, done) - } - function done (err, _mhash) { - if (err) return cb(err) - mhash = _mhash - cb() + multihashing(buf, opts.mhtype, opts.mhlen, cb) } } function generateCid (mhash, block, opts, cb) { - if (opts.format === 'v0') { - cid = new CID(0, 'dag-pb', mhash) - } else { - cid = new CID(1, opts.format, mhash) - } + cid = new CID(opts.verison, opts.format, mhash) cb() } } @@ -73,10 +65,19 @@ module.exports = { mhlen: { describe: 'multihash hash length', default: undefined - } + }, }, handler (argv) { + // parse options + if (argv.format === 'v0') { + argv.verison = 0 + argv.format = 'dag-pb' + argv.mhtype = 'sha2-256' + } else { + argv.verison = 1 + } + if (argv.data) { return addBlock(fs.readFileSync(argv.data), argv) } From 5a00279d123c58e2b11c5eca47166afee64184bf Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 9 Dec 2016 21:01:28 -0800 Subject: [PATCH 7/8] lint - slay nefarious dangling comma, oh woe is me --- src/cli/commands/block/put.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/commands/block/put.js b/src/cli/commands/block/put.js index 5b86f64f93..47210d5012 100644 --- a/src/cli/commands/block/put.js +++ b/src/cli/commands/block/put.js @@ -65,7 +65,7 @@ module.exports = { mhlen: { describe: 'multihash hash length', default: undefined - }, + } }, handler (argv) { From 29bf12e2d1c8ed80559cc7d3d46a3be2fd11b931 Mon Sep 17 00:00:00 2001 From: kumavis Date: Fri, 9 Dec 2016 23:02:47 -0800 Subject: [PATCH 8/8] tests - fix eth-block test data --- test/cli/test-block.js | 2 +- test/test-data/eth-block | Bin 534 -> 534 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli/test-block.js b/test/cli/test-block.js index b52a04e983..308ad3ae9a 100644 --- a/test/cli/test-block.js +++ b/test/cli/test-block.js @@ -19,7 +19,7 @@ describe('block', () => { it('put with flags, format and mhtype', () => { return ipfs('block put --format eth-block --mhtype keccak-256 test/test-data/eth-block').then((out) => { expect(out).to.be.eql( - 'z43AaGF4V81rH6y9ykccadnaGV7EkvbPhFXuvrP6mpK89r8bR5T' + 'z43AaGF23fmvRnDP56Ub9WcJCfzSfqtmzNCCvmz5eudT8dtdCDS' ) }) }) diff --git a/test/test-data/eth-block b/test/test-data/eth-block index 438f4c207e32dc0479771c5986c990128f9f5bd1..f2b366dc93835883562d5cf2d82c70956abaffb5 100644 GIT binary patch delta 49 zcmV-10M7rG1eOG_?ExwA>-(=*zaCc?pnk9M1PJ%^;HlrAy3icK$y{4c5>a)Cd3t;i HG6V)gc-I$p delta 20 bcmbQnGL2=!TgHh86*xO8t7-&|SlFBaQbGpR