diff --git a/Specification.md b/Specification.md index 5b0141998..87ab67897 100644 --- a/Specification.md +++ b/Specification.md @@ -1,17 +1,145 @@ -## Gas limit +# Specification -By default `web3.js` uses `web3.eth.Contract.method.myMethod().estimateGas()`, but this value can differ from the -actual gas that will be used, if the contract you are calling depends on the blockhash, blocknumber or any other source of -randomness making the gas cost nondeterministic. +The CLI should package several "core" extensions. +The API these extensions expose is aimed at `node` environments. +The aim is to provide convenience to power-users & devs. -E.g.: the [`MiniMeToken`](https://github.com/aragon/aragon-apps/blob/master/shared/minime/contracts/MiniMeToken.sol) contract which snapshots balances at certain block numbers. +## IPFS `@aragon/ipfs-utils` -Considering the above-mentioned behavior, the CLI should calculate the recommended gas limit as follows: -- `recommendedGas = estimatedGas`, if `estimatedGas > upperGasLimit` or +Goals: + +- installing & starting a daemon with the right configuration +- interacting with local/remote nodes: authentication, configuration, pinning, etc. +- help setup a "production" node (?) +- automatically pin anything published to an APM repository (?) + +### IPFS Binaries + +Local node: + +- ✔️ `aragon ipfs` - Alias for `aragon ipfs start` +- `aragon ipfs start` - Start the **IPFS Daemon** + - Should start in the background and then finish + - Should start with the recommended configuration from `$HOMEDIR/.aragon/ipfsconfig.json`. E.g.: + + ```json + { + "daemonArgs": [ + "--migrate", + "--enable-namesys-pubsub" + ], + "logsLocation": "$HOMEDIR/.aragon/ipfs-logs" + } + ``` + + - Should save `stdout` and `stderr` to files, e.g.: `stdout-${number}.log` in the `logsLocation` + - Should warn if it's already running + - Should error (exit code 1) if it cannot start (missing libs/ports taken) + - Should inform about where the logs are saved and how to stop it + - Should run `aragon ipfs status` and finish afterwards (optional) +- `aragon ipfs stop` - Stop the **IPFS Daemon** + - Should warn if the node was already stopped +- `aragon ipfs enable-startup` - Start the **IPFS Daemon** automatically at start-up + +Local/remote node: + +- `aragon ipfs status` - Check the configuration of the **IPFS Daemon** + - Should print if CORS is enabled + - Should print which Aragon artifacts are pinned + - Should print number of peers, repository size + - Should print which ports are used + - Should print local ip and public ip + - Should print bootstrapped nodes + - If it's a local node: + - Should print whether 'Run at startup' is enabled + - If it's not configured correctly: + - Should ask whether to run `aragon ipfs configure` + - or Should error (exit code 1) +- `aragon ipfs configure` - Configure the **IPFS Daemon** + - Should configure CORS + - Should pin Aragon artifacts (from ipfs with a fallback to http) + - Should inform the user about advanced configurations with `ipfs config` +- ✔️ `aragon ipfs view ` - Display metadata about the content, such as size, links, etc. +- ✔️ `aragon ipfs propagate ` - Request the content and its links at several gateways, making the files +more distributed within the network + +### IPFS API + +Connection: + +- ✔️ `ensureConnection` + - throws if it cannot be established + - 🚧 auth headers support +- `start` + +Installation: + +- `isInstalled` +- `install` + +Configuration: + +- `hasCORSEnabled` +- `enableCORS` +- `hasAragonArtifacts` +- `listAragonArtifacts` +- `pinAragonArtifacts` +- `hasAuthEnabled` +- `enableAuth` + +Data: + +- ✔️ `getMerkleDAG` +- ✔️ `extractCIDsFromMerkleDAG` +- ✔️ `propagateFiles` + +Data viz: + +- ✔️ `stringifyMerkleDAG` + +## Devchain `@aragon/devchain-utils` + +- Pre-bundled because (is tiny?) + +### Devchain Binaries + +- `aragon devchain` - start ganache with our aragen-generated db + +### Devchain API + +- `ensureConnection` +- `hasAragonDeployements` +- `deployAragon` +- `start` -- should save stdout and stderr in some files to be outputed by `aragon devchain output` + +## Web3 `@aragon/web3-utils` + +Note: perhaps this is better suited for `aragonAPI`. + +### Web3 API + +- `getRecommendedGasLimit` + +By default `web3.js` uses `web3.eth.Contract.method.myMethod().estimateGas()`, but this value can +differ from the actual gas that will be used, if the contract you are calling depends on the +blockhash, blocknumber or any other source of randomness making the gas cost nondeterministic. + +E.g.: the [`MiniMeToken`](https://github.com/aragon/aragon-apps/blob/master/shared/minime/contracts/MiniMeToken.sol) +contract which snapshots balances at certain block numbers. + +Considering the above-mentioned behavior, the CLI should calculate the recommended gas limit +as follows: + +- `recommendedGas = estimatedGas` (if `estimatedGas > upperGasLimit`) or - `recommendedGas = estimatedGas * DEFAULT_GAS_FUZZ_FACTOR` with a maximum value of `upperGasLimit` -Where `upperGasLimit = latestBlock.gasLimit * LAST_BLOCK_GAS_LIMIT_FACTOR`, `DEFAULT_GAS_FUZZ_FACTOR = 1.5` and `LAST_BLOCK_GAS_LIMIT_FACTOR = 0.95`. +Where: + +- `upperGasLimit = latestBlock.gasLimit * LAST_BLOCK_GAS_LIMIT_FACTOR` +- `LAST_BLOCK_GAS_LIMIT_FACTOR = 0.95` +- `DEFAULT_GAS_FUZZ_FACTOR = 1.5` -See `src/util.js#getRecommendedGasLimit`. (This should probably be its own library) +See `src/util.js#getRecommendedGasLimit`. +(This should probably be its own library since `aragon.js` uses it as well) -The CLI should ignore the `gas` property of the network from truffle config. +Note: The CLI should ignore the `gas` property of the network from truffle config. diff --git a/packages/aragon-cli/package.json b/packages/aragon-cli/package.json index d3710a9fd..1b33c9a0b 100644 --- a/packages/aragon-cli/package.json +++ b/packages/aragon-cli/package.json @@ -46,6 +46,7 @@ "@babel/polyfill": "^7.0.0", "ajv": "^6.6.2", "bignumber.js": "^7.1.0", + "byte-size": "^5.0.1", "chalk": "^2.1.0", "cli-table": "^0.3.1", "colors": "^1.2.4", @@ -62,6 +63,7 @@ "go-ipfs": "0.4.17", "ignore": "^3.3.7", "ipfs-api": "^21.0.0", + "ipfs-http-client": "^30.1.0", "js-sha3": "^0.7.0", "listr": "^0.13.0", "listr-input": "0.1.3", @@ -70,10 +72,12 @@ "listr-verbose-renderer": "^0.4.1", "mkdirp": "^0.5.1", "ncp": "^2.0.0", + "node-fetch": "^2.3.0", "opn": "^5.3.0", "rimraf": "^2.6.2", "semver": "^5.4.1", "source-map-support": "^0.5.11", + "stringify-tree": "^1.0.2", "tmp-promise": "^1.0.4", "truffle": "4.1.14", "truffle-flattener": "^1.2.9", diff --git a/packages/aragon-cli/src/commands/apm_cmds/publish.js b/packages/aragon-cli/src/commands/apm_cmds/publish.js index 21d8d4db1..24cb52c3e 100644 --- a/packages/aragon-cli/src/commands/apm_cmds/publish.js +++ b/packages/aragon-cli/src/commands/apm_cmds/publish.js @@ -17,7 +17,7 @@ const execa = require('execa') const { compileContracts } = require('../../helpers/truffle-runner') const web3Utils = require('web3-utils') const deploy = require('../deploy') -const startIPFS = require('../ipfs') +const startIPFS = require('../ipfs_cmds/start') const getRepoTask = require('../dao_cmds/utils/getRepoTask') const listrOpts = require('../../helpers/listr-options') diff --git a/packages/aragon-cli/src/commands/dao_cmds/new.js b/packages/aragon-cli/src/commands/dao_cmds/new.js index a53f76c24..d996a9a51 100644 --- a/packages/aragon-cli/src/commands/dao_cmds/new.js +++ b/packages/aragon-cli/src/commands/dao_cmds/new.js @@ -6,7 +6,7 @@ const chalk = require('chalk') const { getContract } = require('../../util') const getRepoTask = require('./utils/getRepoTask') const listrOpts = require('../../helpers/listr-options') -const startIPFS = require('../ipfs') +const startIPFS = require('../ipfs_cmds/start') const { getRecommendedGasLimit } = require('../../util') exports.BARE_KIT = defaultAPMName('bare-kit') diff --git a/packages/aragon-cli/src/commands/ipfs.js b/packages/aragon-cli/src/commands/ipfs.js index 151707b1f..8284747ef 100644 --- a/packages/aragon-cli/src/commands/ipfs.js +++ b/packages/aragon-cli/src/commands/ipfs.js @@ -1,89 +1,9 @@ -const path = require('path') -const TaskList = require('listr') -const { - startIPFSDaemon, - isIPFSCORS, - setIPFSCORS, - isIPFSRunning, -} = require('../helpers/ipfs-daemon') +const startCommand = require('./ipfs_cmds/start') -const IPFS = require('ipfs-api') -const listrOpts = require('../helpers/listr-options') - -exports.command = 'ipfs' - -exports.describe = 'Start IPFS daemon configured to work with Aragon' - -exports.task = ({ apmOptions, silent, debug }) => { - return new TaskList( - [ - { - title: 'Start IPFS', - task: async (ctx, task) => { - // If the dev manually set their IPFS node, skip install and running check - if (apmOptions.ipfs.rpc.default) { - const running = await isIPFSRunning(apmOptions.ipfs.rpc) - if (!running) { - task.output = 'Starting IPFS at port: ' + apmOptions.ipfs.rpc.port - await startIPFSDaemon() - ctx.started = true - await setIPFSCORS(apmOptions.ipfs.rpc) - } else { - task.output = 'IPFS is started, checking CORS config' - await setIPFSCORS(apmOptions.ipfs.rpc) - return ( - 'Connected to IPFS daemon at port: ' + apmOptions.ipfs.rpc.port - ) - } - } else { - await isIPFSCORS(apmOptions.ipfs.rpc) - return 'Connecting to provided IPFS daemon' - } - }, - }, - { - title: 'Add local files', - task: ctx => { - const ipfs = IPFS('localhost', '5001', { protocol: 'http' }) - const files = path.resolve( - require.resolve('@aragon/aragen'), - '../ipfs-cache' - ) - - return new Promise((resolve, reject) => { - ipfs.util.addFromFs( - files, - { recursive: true, ignore: 'node_modules' }, - (err, files) => { - if (err) return reject(err) - resolve(files) - } - ) - }) - }, - }, - ], - listrOpts(silent, debug) - ) +exports.builder = function(yargs) { + return yargs.commandDir('ipfs_cmds') } -exports.handler = function({ reporter, apm: apmOptions }) { - const task = exports.task({ apmOptions }) - - task - .run() - .then(ctx => { - if (ctx.started) { - reporter.info( - 'IPFS daemon is now running. Stopping this process will stop IPFS' - ) - } else { - reporter.warning('Didnt start IPFS, port busy') - process.exit() - } - }) - .catch(err => { - reporter.error(err) - process.exit(1) - }) -} +exports.command = 'ipfs' +exports.describe = 'Shortcut for aragon ipfs start' +exports.handler = startCommand.handler diff --git a/packages/aragon-cli/src/commands/ipfs_cmds/propagate.js b/packages/aragon-cli/src/commands/ipfs_cmds/propagate.js new file mode 100644 index 000000000..e7dbcdf1e --- /dev/null +++ b/packages/aragon-cli/src/commands/ipfs_cmds/propagate.js @@ -0,0 +1,81 @@ +import TaskList from 'listr' +// +import { + ensureConnection, + getMerkleDAG, + extractCIDsFromMerkleDAG, + propagateFiles, +} from '../../lib/ipfs' +import listrOpts from '../../helpers/listr-options' + +exports.command = 'propagate ' +exports.describe = + 'Request the content and its links at several gateways, making the files more distributed within the network.' + +exports.builder = yargs => { + return yargs.positional('cid', { + description: 'A self-describing content-addressed identifier', + }) +} + +exports.task = ({ apmOptions, silent, debug, cid }) => { + return new TaskList( + [ + { + title: 'Connect to IPFS', + task: async ctx => { + ctx.ipfs = await ensureConnection(apmOptions.ipfs.rpc) + }, + }, + { + title: 'Fetch the links', + task: async ctx => { + ctx.data = await getMerkleDAG(ctx.ipfs.client, cid, { + recursive: true, + }) + }, + }, + { + title: 'Query gateways', + task: async (ctx, task) => { + ctx.CIDs = extractCIDsFromMerkleDAG(ctx.data, { + recursive: true, + }) + + const logger = text => (task.output = text) + ctx.result = await propagateFiles(ctx.CIDs, logger) + }, + }, + ], + listrOpts(silent, debug) + ) +} + +exports.handler = async function({ + reporter, + apm: apmOptions, + cid, + debug, + silent, +}) { + const task = await exports.task({ + reporter, + apmOptions, + cid, + debug, + silent, + }) + + const ctx = await task.run() + + reporter.info( + `Queried ${ctx.CIDs.length} CIDs at ${ctx.result.gateways.length} gateways` + ) + reporter.info(`Requests succeeded: ${ctx.result.succeeded}`) + reporter.info(`Requests failed: ${ctx.result.failed}`) + reporter.debug(`Gateways: ${ctx.result.gateways.join(', ')}`) + reporter.debug( + `Errors: \n${ctx.result.errors.map(JSON.stringify).join('\n')}` + ) + // TODO add your own gateways +} diff --git a/packages/aragon-cli/src/commands/ipfs_cmds/start.js b/packages/aragon-cli/src/commands/ipfs_cmds/start.js new file mode 100644 index 000000000..d5ff8bc48 --- /dev/null +++ b/packages/aragon-cli/src/commands/ipfs_cmds/start.js @@ -0,0 +1,89 @@ +const path = require('path') +const TaskList = require('listr') +const { + startIPFSDaemon, + isIPFSCORS, + setIPFSCORS, + isIPFSRunning, +} = require('../../helpers/ipfs-daemon') + +const IPFS = require('ipfs-api') +const listrOpts = require('../../helpers/listr-options') + +exports.command = 'ipfs start' + +exports.describe = 'Start IPFS daemon configured to work with Aragon' + +exports.task = ({ apmOptions, silent, debug }) => { + return new TaskList( + [ + { + title: 'Start IPFS', + task: async (ctx, task) => { + // If the dev manually set their IPFS node, skip install and running check + if (apmOptions.ipfs.rpc.default) { + const running = await isIPFSRunning(apmOptions.ipfs.rpc) + if (!running) { + task.output = 'Starting IPFS at port: ' + apmOptions.ipfs.rpc.port + await startIPFSDaemon() + ctx.started = true + await setIPFSCORS(apmOptions.ipfs.rpc) + } else { + task.output = 'IPFS is started, checking CORS config' + await setIPFSCORS(apmOptions.ipfs.rpc) + return ( + 'Connected to IPFS daemon at port: ' + apmOptions.ipfs.rpc.port + ) + } + } else { + await isIPFSCORS(apmOptions.ipfs.rpc) + return 'Connecting to provided IPFS daemon' + } + }, + }, + { + title: 'Add local files', + task: ctx => { + const ipfs = IPFS('localhost', '5001', { protocol: 'http' }) + const files = path.resolve( + require.resolve('@aragon/aragen'), + '../ipfs-cache' + ) + + return new Promise((resolve, reject) => { + ipfs.util.addFromFs( + files, + { recursive: true, ignore: 'node_modules' }, + (err, files) => { + if (err) return reject(err) + resolve(files) + } + ) + }) + }, + }, + ], + listrOpts(silent, debug) + ) +} + +exports.handler = function({ reporter, apm: apmOptions }) { + const task = exports.task({ apmOptions }) + + task + .run() + .then(ctx => { + if (ctx.started) { + reporter.info( + 'IPFS daemon is now running. Stopping this process will stop IPFS' + ) + } else { + reporter.warning('Didnt start IPFS, port busy') + process.exit() + } + }) + .catch(err => { + reporter.error(err) + process.exit(1) + }) +} diff --git a/packages/aragon-cli/src/commands/ipfs_cmds/view.js b/packages/aragon-cli/src/commands/ipfs_cmds/view.js new file mode 100644 index 000000000..5ea5dc80b --- /dev/null +++ b/packages/aragon-cli/src/commands/ipfs_cmds/view.js @@ -0,0 +1,61 @@ +import TaskList from 'listr' +// +import { + ensureConnection, + getMerkleDAG, + stringifyMerkleDAG, +} from '../../lib/ipfs' +import listrOpts from '../../helpers/listr-options' + +exports.command = 'view ' +exports.describe = + 'Display metadata about the content, such as size, links, etc.' + +exports.builder = yargs => { + // TODO add support for "ipfs paths", e.g: QmP49YSJVhQTySqLDFTzFZPG8atf3CLsQSPDVj3iATQkhC/arapp.json + return yargs.positional('cid', { + description: 'A self-describing content-addressed identifier', + }) +} + +exports.task = ({ apmOptions, silent, debug, cid }) => { + return new TaskList( + [ + // TODO validation of the CID + { + title: 'Connect to IPFS', + task: async ctx => { + ctx.ipfs = await ensureConnection(apmOptions.ipfs.rpc) + }, + }, + { + title: 'Fetch the links', + task: async ctx => { + ctx.merkleDAG = await getMerkleDAG(ctx.ipfs.client, cid, { + recursive: true, + }) + }, + }, + ], + listrOpts(silent, debug) + ) +} + +exports.handler = async function({ + reporter, + apm: apmOptions, + cid, + debug, + silent, +}) { + const task = await exports.task({ + reporter, + apmOptions, + cid, + debug, + silent, + }) + + const ctx = await task.run() + console.log(stringifyMerkleDAG(ctx.merkleDAG)) +} diff --git a/packages/aragon-cli/src/commands/run.js b/packages/aragon-cli/src/commands/run.js index 7a857b768..aa3adffa7 100644 --- a/packages/aragon-cli/src/commands/run.js +++ b/packages/aragon-cli/src/commands/run.js @@ -6,7 +6,7 @@ const publish = require('./apm_cmds/publish') const devchain = require('./devchain') const deploy = require('./deploy') const newDAO = require('./dao_cmds/new') -const startIPFS = require('./ipfs') +const startIPFS = require('./ipfs_cmds/start') const encodeInitPayload = require('./dao_cmds/utils/encodeInitPayload') const { promisify } = require('util') const clone = promisify(require('git-clone')) diff --git a/packages/aragon-cli/src/lib/ipfs/index.js b/packages/aragon-cli/src/lib/ipfs/index.js new file mode 100644 index 000000000..4428b6db6 --- /dev/null +++ b/packages/aragon-cli/src/lib/ipfs/index.js @@ -0,0 +1,169 @@ +import chalk from 'chalk' +import byteSize from 'byte-size' +import { stringifyTree } from 'stringify-tree' +import ipfsAPI from 'ipfs-http-client' // TODO: import only submodules? +import fetch from 'node-fetch' + +const FETCH_TIMEOUT = 20000 // 20s +const FETCH_TIMEOUT_ERR = 'Request timed out' + +export async function ensureConnection(rpc) { + try { + const client = connectToAPI(rpc) + await client.id() + return { + client, + } + } catch (e) { + throw new Error(`Could not connect to IPFS at ${JSON.stringify(rpc)}`) + } +} + +export function connectToAPI(rpc) { + return ipfsAPI(rpc) +} + +export async function getMerkleDAG(client, cid, opts = {}) { + const merkleDAG = parseMerkleDAG(await client.object.get(cid)) + merkleDAG.cid = cid + + if (opts.recursive && merkleDAG.isDir && merkleDAG.links) { + // fetch the MerkleDAG of each link recursively + const promises = merkleDAG.links.map(async link => { + const object = await getMerkleDAG(client, link.cid, opts) + return Object.assign(link, object) + }) + + return Promise.all(promises).then(links => { + merkleDAG.links = links + return merkleDAG + }) + } + + return merkleDAG +} + +// object.get returns an object of type DAGNode +// https://github.com/ipld/js-ipld-dag-pb#dagnode-instance-methods-and-properties +function parseMerkleDAG(dagNode) { + const parsed = dagNode.toJSON() + // add relevant data + parsed.isDir = isDir(parsed.data) + // remove irrelevant data + delete parsed.data + if (!parsed.isDir) { + // if it's a big file it will have links to its other chunks + delete parsed.links + } + return parsed +} + +function isDir(data) { + return data.length === 2 && data.toString() === '\u0008\u0001' +} + +function stringifyMerkleDAGNode(merkleDAG) { + // ${merkleDAG.isDir ? '📁' : ''} + const cid = merkleDAG.cid + const name = merkleDAG.name || 'root' + const parsedSize = byteSize(merkleDAG.size) + const size = parsedSize.value + parsedSize.unit + const delimiter = chalk.gray(' - ') + + return [name, size, chalk.gray(cid)].join(delimiter) +} + +export function stringifyMerkleDAG(merkleDAG) { + return stringifyTree( + merkleDAG, + node => stringifyMerkleDAGNode(node), + node => node.links + ) +} + +export function extractCIDsFromMerkleDAG(merkleDAG, opts = {}) { + const CIDs = [] + CIDs.push(merkleDAG.cid) + + if (opts.recursive && merkleDAG.isDir && merkleDAG.links) { + merkleDAG.links + .map(merkleDAGOfLink => extractCIDsFromMerkleDAG(merkleDAGOfLink, opts)) + .map(CIDsOfLink => CIDs.push(...CIDsOfLink)) + } + + return CIDs +} + +function timeout() { + return new Promise((resolve, reject) => { + setTimeout(() => { + reject(FETCH_TIMEOUT_ERR) + }, FETCH_TIMEOUT) + }) +} + +const gateways = [ + 'https://ipfs.io/ipfs', + 'https://ipfs.infura.io/ipfs', + 'https://cloudflare-ipfs.com/ipfs', + 'https://ipfs.eth.aragon.network/ipfs', + 'https://ipfs.jes.xxx/ipfs', + 'https://www.eternum.io/ipfs', + 'https://ipfs.wa.hle.rs/ipfs', +] + +async function queryCidAtGateway(gateway, cid) { + try { + await Promise.race([ + fetch(`${gateway}/${cid}`), + // Add a timeout because the Fetch API does not implement them + timeout(), + ]) + + return { + success: true, + cid, + gateway, + } + } catch (err) { + return { + success: false, + cid, + gateway, + error: err, + } + } +} + +async function propagateFile(cid, logger) { + const results = await Promise.all( + gateways.map(gateway => queryCidAtGateway(gateway, cid)) + ) + + const succeeded = results.filter(status => status.success).length + const failed = gateways.length - succeeded + + logger( + `Queried ${cid} at ${succeeded} gateways successfully, ${failed} failed.` + ) + + const errors = results + .filter(result => result.error) + .map(result => result.error) + + return { + succeeded, + failed, + errors, + } +} + +export async function propagateFiles(CIDs, logger = () => {}) { + const results = await Promise.all(CIDs.map(cid => propagateFile(cid, logger))) + return { + gateways, + succeeded: results.reduce((prev, current) => prev + current.succeeded, 0), + failed: results.reduce((prev, current) => prev + current.failed, 0), + errors: results.reduce((prev, current) => [...prev, ...current.errors], []), + } +} diff --git a/packages/e2e-tests/src/cli/help.js.md b/packages/e2e-tests/src/cli/help.js.md index dfc37e3e6..48a5d2995 100644 --- a/packages/e2e-tests/src/cli/help.js.md +++ b/packages/e2e-tests/src/cli/help.js.md @@ -28,8 +28,7 @@ Generated by [AVA](https://ava.li). aragon init [template] (deprecated) Initialise a new application.␊ Deprecated in favor of `npx create-aragon-app␊ [template]`␊ - aragon ipfs Start IPFS daemon configured to work with␊ - Aragon␊ + aragon ipfs Shortcut for aragon ipfs start␊ aragon run Run the current app locally␊ ␊ APM:␊ diff --git a/packages/e2e-tests/src/cli/help.js.snap b/packages/e2e-tests/src/cli/help.js.snap index 99ec15dfc..073d444c6 100644 Binary files a/packages/e2e-tests/src/cli/help.js.snap and b/packages/e2e-tests/src/cli/help.js.snap differ