From 905bdc03edfba23d35aa6a5b8e356ab56f6f36b1 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 15 Feb 2018 11:02:22 +0000 Subject: [PATCH] feat: stats API (stats.bitswap and stats.repo) (#1198) --- README.md | 16 ++++++++---- package.json | 4 ++- src/cli/commands/bitswap/stat.js | 20 +++++++-------- src/cli/commands/repo/stat.js | 31 +++++++++++++++++++++++ src/cli/commands/repo/version.js | 3 ++- src/cli/commands/stats.js | 14 +++++++++++ src/cli/commands/stats/bitswap.js | 37 ++++++++++++++++++++++++++++ src/cli/commands/stats/repo.js | 31 +++++++++++++++++++++++ src/core/components/bitswap.js | 24 ++++++++++++++---- src/core/components/index.js | 1 + src/core/components/repo.js | 27 +++++++++++++++++--- src/core/components/start.js | 2 +- src/core/components/stats.js | 8 ++++++ src/core/index.js | 1 + src/http/api/resources/bitswap.js | 34 +++++++++++++++---------- src/http/api/resources/index.js | 1 + src/http/api/resources/repo.js | 41 +++++++++++++++++++++++++++++++ src/http/api/resources/stats.js | 7 ++++++ src/http/api/routes/index.js | 3 ++- src/http/api/routes/repo.js | 17 ++++++++++--- src/http/api/routes/stats.js | 23 +++++++++++++++++ test/cli/bitswap.js | 3 +-- test/cli/commands.js | 2 +- 23 files changed, 304 insertions(+), 46 deletions(-) create mode 100644 src/cli/commands/repo/stat.js create mode 100644 src/cli/commands/stats.js create mode 100644 src/cli/commands/stats/bitswap.js create mode 100644 src/cli/commands/stats/repo.js create mode 100644 src/core/components/stats.js create mode 100644 src/http/api/resources/stats.js create mode 100644 src/http/api/routes/stats.js diff --git a/README.md b/README.md index e82d81d104..7aa47baf01 100644 --- a/README.md +++ b/README.md @@ -256,11 +256,6 @@ A complete API definition is in the works. Meanwhile, you can learn how to you u - [`ipfs.block.put(block, cid, [callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/BLOCK.md#put) - [`ipfs.block.stat(cid, [callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/BLOCK.md#stat) -- [repo](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/) - - `ipfs.repo.init` - - `ipfs.repo.version` - - `ipfs.repo.gc` (not implemented, yet!) - #### `Graph` - [dag](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/DAG.md) @@ -334,6 +329,17 @@ A complete API definition is in the works. Meanwhile, you can learn how to you u - `ipfs.start([callback])` - [`ipfs.stop([callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/MISCELLANEOUS.md#stop) - `ipfs.isOnline()` + +- [repo](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/REPO.md) + - `ipfs.repo.init` + - [`ipfs.repo.stat([options, callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/REPO.md#stat) + - [`ipfs.repo.version([callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/REPO.md#version) + - `ipfs.repo.gc([options, callback])` (not implemented, yet!) + +- [stats](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/STATS.md) + - [`ipfs.stats.bitswap([callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/STATS.md#bitswap) + - [`ipfs.stats.bw([options, callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/STATS.md#bw) (not implemented, yet!) + - [`ipfs.stats.repo([options, callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/STATS.md#repo) - [config](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/CONFIG.md) - [`ipfs.config.get([key, callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/CONFIG.md#configget) diff --git a/package.json b/package.json index d07e1d9694..d927d3913d 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "form-data": "^2.3.2", "go-ipfs-dep": "^0.4.13", "hat": "0.0.3", - "interface-ipfs-core": "~0.51.0", + "interface-ipfs-core": "~0.52.0", "ipfsd-ctl": "~0.28.0", "left-pad": "^1.2.0", "lodash": "^4.17.5", @@ -90,6 +90,7 @@ }, "dependencies": { "async": "^2.6.0", + "big.js": "^5.0.3", "binary-querystring": "~0.1.2", "bl": "^1.2.1", "boom": "^7.1.1", @@ -107,6 +108,7 @@ "hoek": "^5.0.3", "ipfs-api": "^18.0.0", "ipfs-bitswap": "~0.19.0", + "human-to-milliseconds": "^1.0.0", "ipfs-block": "~0.6.1", "ipfs-block-service": "~0.13.0", "ipfs-multipart": "~0.1.0", diff --git a/src/cli/commands/bitswap/stat.js b/src/cli/commands/bitswap/stat.js index ef55ec0d1c..081dceb595 100644 --- a/src/cli/commands/bitswap/stat.js +++ b/src/cli/commands/bitswap/stat.js @@ -16,22 +16,22 @@ module.exports = { throw err } - stats.Wantlist = stats.Wantlist || [] - stats.Wantlist = stats.Wantlist.map((entry) => { + stats.wantlist = stats.wantlist || [] + stats.wantlist = stats.wantlist.map((entry) => { const buf = Buffer.from(entry.cid.hash.data) const cid = new CID(entry.cid.version, entry.cid.codec, buf) return cid.toBaseEncodedString() }) - stats.Peers = stats.Peers || [] + stats.peers = stats.peers || [] print(`bitswap status - blocks received: ${stats.BlocksReceived} - dup blocks received: ${stats.DupBlksReceived} - dup data received: ${stats.DupDataReceived}B - wantlist [${stats.Wantlist.length} keys] - ${stats.Wantlist.join('\n ')} - partners [${stats.Peers.length}] - ${stats.Peers.join('\n ')}`) + blocks received: ${stats.blocksReceived} + dup blocks received: ${stats.dupBlksReceived} + dup data received: ${stats.dupDataReceived}B + wantlist [${stats.wantlist.length} keys] + ${stats.wantlist.join('\n ')} + partners [${stats.peers.length}] + ${stats.peers.join('\n ')}`) }) } } diff --git a/src/cli/commands/repo/stat.js b/src/cli/commands/repo/stat.js new file mode 100644 index 0000000000..27f3f46ed1 --- /dev/null +++ b/src/cli/commands/repo/stat.js @@ -0,0 +1,31 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'stat', + + describe: 'Get stats for the currently used repo', + + builder: { + human: { + type: 'boolean', + default: false + } + }, + + handler (argv) { + argv.ipfs.repo.stat({human: argv.human}, (err, stats) => { + if (err) { + throw err + } + + print(`repo status + number of objects: ${stats.numObjects} + repo size: ${stats.repoSize} + repo path: ${stats.repoPath} + version: ${stats.version} + maximum storage: ${stats.storageMax}`) + }) + } +} diff --git a/src/cli/commands/repo/version.js b/src/cli/commands/repo/version.js index 462a571da2..9a0f5271a8 100644 --- a/src/cli/commands/repo/version.js +++ b/src/cli/commands/repo/version.js @@ -10,10 +10,11 @@ module.exports = { builder: {}, handler (argv) { - argv.ipfs.repo.version(function (err, version) { + argv.ipfs.repo.version((err, version) => { if (err) { throw err } + print(version) }) } diff --git a/src/cli/commands/stats.js b/src/cli/commands/stats.js new file mode 100644 index 0000000000..4eaa9adaa6 --- /dev/null +++ b/src/cli/commands/stats.js @@ -0,0 +1,14 @@ +'use strict' + +module.exports = { + command: 'stats ', + + description: 'Query IPFS statistics.', + + builder (yargs) { + return yargs.commandDir('stats') + }, + + handler (argv) { + } +} diff --git a/src/cli/commands/stats/bitswap.js b/src/cli/commands/stats/bitswap.js new file mode 100644 index 0000000000..b5c4fe48f1 --- /dev/null +++ b/src/cli/commands/stats/bitswap.js @@ -0,0 +1,37 @@ +'use strict' + +const CID = require('cids') +const print = require('../../utils').print + +module.exports = { + command: 'bitswap', + + describe: 'Show some diagnostic information on the bitswap agent.', + + builder: {}, + + handler (argv) { + argv.ipfs.stats.bitswap((err, stats) => { + if (err) { + throw err + } + + stats.wantlist = stats.wantlist || [] + stats.wantlist = stats.wantlist.map((entry) => { + const buf = Buffer.from(entry.cid.hash.data) + const cid = new CID(entry.cid.version, entry.cid.codec, buf) + return cid.toBaseEncodedString() + }) + stats.peers = stats.peers || [] + + print(`bitswap status + blocks received: ${stats.blocksReceived} + dup blocks received: ${stats.dupBlksReceived} + dup data received: ${stats.dupDataReceived}B + wantlist [${stats.wantlist.length} keys] + ${stats.wantlist.join('\n ')} + partners [${stats.peers.length}] + ${stats.peers.join('\n ')}`) + }) + } +} diff --git a/src/cli/commands/stats/repo.js b/src/cli/commands/stats/repo.js new file mode 100644 index 0000000000..366f2ae719 --- /dev/null +++ b/src/cli/commands/stats/repo.js @@ -0,0 +1,31 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'repo', + + describe: 'Get stats for the currently used repo', + + builder: { + human: { + type: 'boolean', + default: false + } + }, + + handler (argv) { + argv.ipfs.stats.repo({human: argv.human}, (err, stats) => { + if (err) { + throw err + } + + print(`repo status + number of objects: ${stats.numObjects} + repo size: ${stats.repoSize} + repo path: ${stats.repoPath} + version: ${stats.version} + maximum storage: ${stats.storageMax}`) + }) + } +} diff --git a/src/core/components/bitswap.js b/src/core/components/bitswap.js index 6a1bffefc2..e148f8c165 100644 --- a/src/core/components/bitswap.js +++ b/src/core/components/bitswap.js @@ -1,6 +1,9 @@ 'use strict' const OFFLINE_ERROR = require('../utils').OFFLINE_ERROR +const promisify = require('promisify-es6') +const setImmediate = require('async/setImmediate') +const Big = require('big.js') function formatWantlist (list) { return Array.from(list).map((e) => e[1]) @@ -16,16 +19,27 @@ module.exports = function bitswap (self) { const list = self._bitswap.getWantlist() return formatWantlist(list) }, - stat: () => { + + stat: promisify((callback) => { if (!self.isOnline()) { - throw new Error(OFFLINE_ERROR) + return setImmediate(() => callback(new Error(OFFLINE_ERROR))) } - return Object.assign({}, self._bitswap.stat().snapshot, { + const snapshot = self._bitswap.stat().snapshot + + callback(null, { + provideBufLen: parseInt(snapshot.providesBufferLength.toString()), + blocksReceived: new Big(snapshot.blocksReceived), wantlist: formatWantlist(self._bitswap.getWantlist()), - peers: self._bitswap.peers().map((id) => id.toB58String()) + peers: self._bitswap.peers().map((id) => id.toB58String()), + dupBlksReceived: new Big(snapshot.dupBlksReceived), + dupDataReceived: new Big(snapshot.dupDataReceived), + dataReceived: new Big(snapshot.dataReceived), + blocksSent: new Big(snapshot.blocksSent), + dataSent: new Big(snapshot.dataSent) }) - }, + }), + unwant: (key) => { if (!self.isOnline()) { throw new Error(OFFLINE_ERROR) diff --git a/src/core/components/index.js b/src/core/components/index.js index 4aea90db9e..4f4e410e43 100644 --- a/src/core/components/index.js +++ b/src/core/components/index.js @@ -22,3 +22,4 @@ exports.pubsub = require('./pubsub') exports.dht = require('./dht') exports.dns = require('./dns') exports.key = require('./key') +exports.stats = require('./stats') diff --git a/src/core/components/repo.js b/src/core/components/repo.js index eb488346a9..989e07fd9e 100644 --- a/src/core/components/repo.js +++ b/src/core/components/repo.js @@ -1,16 +1,37 @@ 'use strict' +const promisify = require('promisify-es6') + module.exports = function repo (self) { return { init: (bits, empty, callback) => { // 1. check if repo already exists }, - version: (callback) => { + version: promisify((callback) => { self._repo.version.get(callback) - }, + }), + + gc: () => {}, + + stat: promisify((options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + + self._repo.stat(options, (err, stats) => { + if (err) return callback(err) - gc: function () {}, + callback(null, { + numObjects: stats.numObjects, + repoSize: stats.repoSize, + repoPath: stats.repoPath, + version: stats.version.toString(), + storageMax: stats.storageMax + }) + }) + }), path: () => self._repo.path } diff --git a/src/core/components/start.js b/src/core/components/start.js index 9575c3e3c6..3004cca34d 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -45,7 +45,7 @@ module.exports = (self) => { self._bitswap = new Bitswap( self._libp2pNode, self._repo.blocks, - self._peerInfoBook + { statsEnabled: true } ) self._bitswap.start() diff --git a/src/core/components/stats.js b/src/core/components/stats.js new file mode 100644 index 0000000000..6bc7121301 --- /dev/null +++ b/src/core/components/stats.js @@ -0,0 +1,8 @@ +'use strict' + +module.exports = function stats (self) { + return { + bitswap: require('./bitswap')(self).stat, + repo: require('./repo')(self).stat + } +} diff --git a/src/core/index.js b/src/core/index.js index 83cc20cd37..2d0da3e53f 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -97,6 +97,7 @@ class IPFS extends EventEmitter { this.dht = components.dht(this) this.dns = components.dns(this) this.key = components.key(this) + this.stats = components.stats(this) if (this._options.EXPERIMENTAL.pubsub) { this.log('EXPERIMENTAL pubsub is enabled') diff --git a/src/http/api/resources/bitswap.js b/src/http/api/resources/bitswap.js index ed749ea004..da830c6415 100644 --- a/src/http/api/resources/bitswap.js +++ b/src/http/api/resources/bitswap.js @@ -21,19 +21,27 @@ exports.wantlist = (request, reply) => { } exports.stat = (request, reply) => { - let stats - try { - stats = request.server.app.ipfs.bitswap.stat() - } catch (err) { - return reply(boom.badRequest(err)) - } - - reply({ - BlocksReceived: stats.blocksReceived, - Wantlist: stats.wantlist, - Peers: stats.peers, - DupBlksReceived: stats.dupBlksReceived, - DupDataReceived: stats.dupDataReceived + const ipfs = request.server.app.ipfs + + ipfs.bitswap.stat((err, stats) => { + if (err) { + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + reply({ + ProvideBufLen: stats.provideBufLen, + BlocksReceived: stats.blocksReceived, + Wantlist: stats.wantlist, + Peers: stats.peers, + DupBlksReceived: stats.dupBlksReceived, + DupDataReceived: stats.dupDataReceived, + DataReceived: stats.dataReceived, + BlocksSent: stats.blocksSent, + DataSent: stats.dataSent + }) }) } diff --git a/src/http/api/resources/index.js b/src/http/api/resources/index.js index efc7785dbd..08d8d7f2a1 100644 --- a/src/http/api/resources/index.js +++ b/src/http/api/resources/index.js @@ -15,3 +15,4 @@ exports.files = require('./files') exports.pubsub = require('./pubsub') exports.dns = require('./dns') exports.key = require('./key') +exports.stats = require('./stats') diff --git a/src/http/api/resources/repo.js b/src/http/api/resources/repo.js index ccacec309b..394fda548e 100644 --- a/src/http/api/resources/repo.js +++ b/src/http/api/resources/repo.js @@ -1 +1,42 @@ 'use strict' + +exports = module.exports + +exports.version = (request, reply) => { + const ipfs = request.server.app.ipfs + + ipfs.repo.version((err, version) => { + if (err) { + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + reply({ + Version: version + }) + }) +} + +exports.stat = (request, reply) => { + const ipfs = request.server.app.ipfs + const human = request.query.human === 'true' + + ipfs.repo.stat({human: human}, (err, stat) => { + if (err) { + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + reply({ + NumObjects: stat.numObjects, + RepoSize: stat.repoSize, + RepoPath: stat.repoPath, + Version: stat.version, + StorageMax: stat.storageMax + }) + }) +} diff --git a/src/http/api/resources/stats.js b/src/http/api/resources/stats.js new file mode 100644 index 0000000000..839f92a3b3 --- /dev/null +++ b/src/http/api/resources/stats.js @@ -0,0 +1,7 @@ +'use strict' + +exports = module.exports + +exports.bitswap = require('./bitswap').stat + +exports.repo = require('./repo').stat diff --git a/src/http/api/routes/index.js b/src/http/api/routes/index.js index aeba5f1427..2eeb4b137a 100644 --- a/src/http/api/routes/index.js +++ b/src/http/api/routes/index.js @@ -7,7 +7,7 @@ module.exports = (server) => { require('./bootstrap')(server) require('./block')(server) require('./object')(server) - // require('./repo')(server) + require('./repo')(server) require('./config')(server) require('./swarm')(server) require('./bitswap')(server) @@ -18,4 +18,5 @@ module.exports = (server) => { require('./webui')(server) require('./dns')(server) require('./key')(server) + require('./stats')(server) } diff --git a/src/http/api/routes/repo.js b/src/http/api/routes/repo.js index f42f03bfc1..3eb7ab2a88 100644 --- a/src/http/api/routes/repo.js +++ b/src/http/api/routes/repo.js @@ -2,13 +2,24 @@ const resources = require('./../resources') -// TODO module.exports = (server) => { const api = server.select('API') api.route({ method: '*', - path: '/api/v0/repo', - handler: resources.repo + path: '/api/v0/repo/version', + config: { + handler: resources.repo.version + } }) + + api.route({ + method: '*', + path: '/api/v0/repo/stat', + config: { + handler: resources.repo.stat + } + }) + + // TODO: implement the missing spec https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/REPO.md } diff --git a/src/http/api/routes/stats.js b/src/http/api/routes/stats.js new file mode 100644 index 0000000000..c167ec58c0 --- /dev/null +++ b/src/http/api/routes/stats.js @@ -0,0 +1,23 @@ +'use strict' + +const resources = require('./../resources') + +module.exports = (server) => { + const api = server.select('API') + + api.route({ + method: '*', + path: '/api/v0/stats/bitswap', + config: { + handler: resources.stats.bitswap + } + }) + + api.route({ + method: '*', + path: '/api/v0/stats/repo', + config: { + handler: resources.stats.repo + } + }) +} diff --git a/test/cli/bitswap.js b/test/cli/bitswap.js index 6652364f9b..29220f7b71 100644 --- a/test/cli/bitswap.js +++ b/test/cli/bitswap.js @@ -23,8 +23,7 @@ describe('bitswap', () => runOn((thing) => { }) }) - // TODO @hacdias fix this with https://github.com/ipfs/js-ipfs/pull/1198 - it.skip('stat', function () { + it('stat', function () { this.timeout(20 * 1000) return ipfs('bitswap stat').then((out) => { diff --git a/test/cli/commands.js b/test/cli/commands.js index 50afc3240e..d1d0812957 100644 --- a/test/cli/commands.js +++ b/test/cli/commands.js @@ -4,7 +4,7 @@ const expect = require('chai').expect const runOnAndOff = require('../utils/on-and-off') -const commandCount = 68 +const commandCount = 72 describe('commands', () => runOnAndOff((thing) => { let ipfs