From afd569d4b23c29ee7aabd4e432e782dfd1ba1053 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 11 Jul 2023 13:19:18 -0700 Subject: [PATCH] test: create add-dir benchmark (#167) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: create add-dir benchmark * chore(bench/add-dir): remove unused taskResult prop * test(bench/add-dir): add kubo-direct test * docs(bench/add-dir): ipfs-core node_modules output * chore(bench/add-dir): kubo-direct doesnt pin files when adding * chore(bench/add-dir): split helia benchmark by blockstore type (fs/mem) * fix: ⚡ Double Digit Percentage Improvements. (#169) * fix: :zap: 2x improvement Signed-off-by: Nishant Arora <1895906+whizzzkid@users.noreply.github.com> * fix: dag generation with parallelization at each level --------- Signed-off-by: Nishant Arora <1895906+whizzzkid@users.noreply.github.com> Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> * test: use globSource in add-dir benchmark (#171) * fix: use glob source for importing directories - Use `globSource` from ipfs-utils for import - Make all impls return the same CID - Remove the dir size - if the CID is the same that's good enough * chore: fix linting * chore: remove benchmark from workspaces * test: align error and success output headers --------- Signed-off-by: Nishant Arora <1895906+whizzzkid@users.noreply.github.com> Co-authored-by: Nishant Arora <1895906+whizzzkid@users.noreply.github.com> Co-authored-by: Alex Potsides --- benchmarks/add-dir/README.md | 193 ++++++++++++++++++++++++++ benchmarks/add-dir/package.json | 37 +++++ benchmarks/add-dir/src/helia.ts | 78 +++++++++++ benchmarks/add-dir/src/index.ts | 182 ++++++++++++++++++++++++ benchmarks/add-dir/src/ipfs.ts | 46 ++++++ benchmarks/add-dir/src/kubo-direct.ts | 30 ++++ benchmarks/add-dir/src/kubo.ts | 46 ++++++ benchmarks/add-dir/tsconfig.json | 18 +++ 8 files changed, 630 insertions(+) create mode 100644 benchmarks/add-dir/README.md create mode 100644 benchmarks/add-dir/package.json create mode 100644 benchmarks/add-dir/src/helia.ts create mode 100644 benchmarks/add-dir/src/index.ts create mode 100644 benchmarks/add-dir/src/ipfs.ts create mode 100644 benchmarks/add-dir/src/kubo-direct.ts create mode 100644 benchmarks/add-dir/src/kubo.ts create mode 100644 benchmarks/add-dir/tsconfig.json diff --git a/benchmarks/add-dir/README.md b/benchmarks/add-dir/README.md new file mode 100644 index 00000000..2e2b2174 --- /dev/null +++ b/benchmarks/add-dir/README.md @@ -0,0 +1,193 @@ +# addDir Benchmark + +Benchmarks Helia addDir performance against js-ipfs and Kubo + +- adds 3 separate directories: /src, /dist, /../gc/src +- saves the final CID for each added directory to ensure the CID constructed is the same +- re-instantiates a new instance for each test +- completely removes all files created in repoPath after each test +- disables pinning when using js-ipfs and kubo (it's usually enabled by default) + +All three implementations use on-disk block/datastores to ensure a reasonable basis for comparison. + +Warning! It can take a long time to run against folders with a very large number of files - on the order of a whole day. + +Example, running against `/node_modules` takes Helia 13 minutes for a single run. + +You can speed things up by removing js-ipfs from the `impls` array. + +To run: + +1. Add `benchmarks/*` to the `workspaces` entry in the root `package.json` of this repo +2. Run + ```console + $ npm run reset + $ npm i + $ npm run build + $ cd benchmarks/add-dir + $ npm start + + > benchmarks-gc@1.0.0 start + > npm run build && node dist/src/index.js + + + > benchmarks-gc@1.0.0 build + > aegir build --bundle false + + [14:51:28] tsc [started] + [14:51:33] tsc [completed] + generating Ed25519 keypair... + ┌─────────┬─────────────────────┬─────────┬───────────┬──────┬───────────┬───────────────────────────────────────────────────────────────┐ + │ (index) │ Implementation │ ops/s │ ms/op │ runs │ p99 │ CID │ + ├─────────┼─────────────────────┼─────────┼───────────┼──────┼───────────┼───────────────────────────────────────────────────────────────┤ + //... results here + ``` + +## Things to improve + +- [ ] Process hangs sometimes (hanging promises not resolving...?). Temporarily fixed with `process.exit(0)` after test results are output +- [ ] The Helia test is generating a different CID than js-ipfs & Kubo. This is expected because defaults configured are different, but for accurate comparisons, we should ensure the same structure is generated. +- [ ] Cannot test using `TEST_PATH=node_modules` yet (test takes roughly 13 minutes for a single run, even when running only helia and altering fileImport and blockWrite concurrency) +- [ ] Instead of tearing down the entire instance of each implementation, we may want to simply delete the entries in the blockstore (i.e. afterEach: cleanBlockstore()) +- [ ] we should be able to generate a set of files/folders with a specific count, and total size, that we can run these tests against. (using https://github.com/jbenet/go-random-files ?) +- [ ] we should add an implementation where we execute kubo directly instead of through ipfsd-ctl + +## Changing the benchmark + +You can set environment variables to change how the benchmark is ran + +### ITERATIONS + +Change the number of iterations that are ran +```bash +ITERATIONS=1 npm start + +> benchmarks-add-dir@1.0.0 start +> npm run build && node dist/src/index.js + + +> benchmarks-add-dir@1.0.0 build +> aegir build --bundle false + +[13:00:02] tsc [started] +[13:00:03] tsc [completed] +generating Ed25519 keypair... +(node:12805) ExperimentalWarning: The Fetch API is an experimental feature. This feature could change at any time +(Use `node --trace-warnings ...` to show where the warning was created) +generating Ed25519 keypair... +generating Ed25519 keypair... +┌─────────┬───────────────────────────┬──────────┬──────────┬──────┬───────────┬───────────────────────────────────────────────────────────────┐ +│ (index) │ Implementation │ ops/s │ ms/op │ runs │ p99 │ CID │ +├─────────┼───────────────────────────┼──────────┼──────────┼──────┼───────────┼───────────────────────────────────────────────────────────────┤ +│ 0 │ 'helia-fs - src' │ '438.44' │ '2.28' │ 439 │ '3.97' │ 'bafybeifaymukvfkyw6xgh4th7tsctiifr4ea2btoznf46y6b2fnvikdczi' │ +│ 1 │ 'helia-fs - dist' │ '81.62' │ '12.25' │ 82 │ '316.61' │ 'bafybeibm6mdqrw34ipb7r5pzam6jod5behsoxs73upv3zvucpq2zlv6x2i' │ +│ 2 │ 'helia-fs - ../gc/src' │ '475.87' │ '2.10' │ 476 │ '5.59' │ 'bafybeihhyvzl4zqbvvtafd6cnp37gwvrypn2cxpyr2yj5zppvgk3urxgpm' │ +│ 3 │ 'helia-mem - src' │ '924.29' │ '1.08' │ 925 │ '2.08' │ 'bafybeifaymukvfkyw6xgh4th7tsctiifr4ea2btoznf46y6b2fnvikdczi' │ +│ 4 │ 'helia-mem - dist' │ '242.95' │ '4.12' │ 243 │ '5.92' │ 'bafybeibm6mdqrw34ipb7r5pzam6jod5behsoxs73upv3zvucpq2zlv6x2i' │ +│ 5 │ 'helia-mem - ../gc/src' │ '901.17' │ '1.11' │ 902 │ '2.70' │ 'bafybeihhyvzl4zqbvvtafd6cnp37gwvrypn2cxpyr2yj5zppvgk3urxgpm' │ +│ 6 │ 'ipfs - src' │ '11.60' │ '86.22' │ 12 │ '103.93' │ 'bafybeihumrxwxpovdza7v7ukatwjye3ylpsrzx3sou2vw4s7zyjm55vdxy' │ +│ 7 │ 'ipfs - dist' │ '2.29' │ '436.19' │ 5 │ '456.56' │ 'bafybeiflrkun45ltbm5zg3uj2uw2nhtgj7rplcnshyo5domndyjbmp2xzy' │ +│ 8 │ 'ipfs - ../gc/src' │ '6.11' │ '163.68' │ 7 │ '181.44' │ 'bafybeibdpig6o56rjems2twzgvog7ssatt5szrpnjgvtnws4i4bm5csvoa' │ +│ 9 │ 'kubo - src' │ '13.78' │ '72.56' │ 14 │ '248.15' │ 'bafybeihumrxwxpovdza7v7ukatwjye3ylpsrzx3sou2vw4s7zyjm55vdxy' │ +│ 10 │ 'kubo - dist' │ '2.50' │ '400.71' │ 5 │ '1208.74' │ 'bafybeiflrkun45ltbm5zg3uj2uw2nhtgj7rplcnshyo5domndyjbmp2xzy' │ +│ 11 │ 'kubo - ../gc/src' │ '11.79' │ '84.79' │ 12 │ '433.94' │ 'bafybeifqlusi6zeboi7mxdbbjr5y5pdojrohhtelm4rbhb2vfkfa6f2kfu' │ +│ 12 │ 'kubo-direct - src' │ '14.68' │ '68.13' │ 15 │ '240.11' │ 'bafybeihumrxwxpovdza7v7ukatwjye3ylpsrzx3sou2vw4s7zyjm55vdxy' │ +│ 13 │ 'kubo-direct - dist' │ '2.48' │ '403.53' │ 5 │ '1232.42' │ 'bafybeiflrkun45ltbm5zg3uj2uw2nhtgj7rplcnshyo5domndyjbmp2xzy' │ +│ 14 │ 'kubo-direct - ../gc/src' │ '13.89' │ '71.99' │ 14 │ '329.82' │ 'bafybeifqlusi6zeboi7mxdbbjr5y5pdojrohhtelm4rbhb2vfkfa6f2kfu' │ +└─────────┴───────────────────────────┴──────────┴──────────┴──────┴───────────┴───────────────────────────────────────────────────────────────┘ +``` + +### MIN_TIME + +Change how long we want to allow the test to run, running as many iterations as possible +```bash +MIN_TIME=1000 npm start + +> benchmarks-add-dir@1.0.0 start +> npm run build && node dist/src/index.js + + +> benchmarks-add-dir@1.0.0 build +> aegir build --bundle false + +[13:00:38] tsc [started] +[13:00:39] tsc [completed] +generating Ed25519 keypair... +(node:13367) ExperimentalWarning: The Fetch API is an experimental feature. This feature could change at any time +(Use `node --trace-warnings ...` to show where the warning was created) +generating Ed25519 keypair... +generating Ed25519 keypair... +┌─────────┬─────────────────────┬──────────┬──────────┬──────┬───────────┬───────────────────────────────────────────────────────────────┐ +│ (index) │ Implementation │ ops/s │ ms/op │ runs │ p99 │ CID │ +├─────────┼─────────────────────┼──────────┼──────────┼──────┼───────────┼───────────────────────────────────────────────────────────────┤ +│ 0 │ 'helia - src' │ '516.16' │ '1.94' │ 517 │ '3.47' │ 'bafybeievc57xgecd6icfsrp2v4t2a5fepicezabwcvh7javhx7gr7zkbnu' │ +│ 1 │ 'helia - dist' │ '126.64' │ '7.90' │ 127 │ '13.49' │ 'bafybeiewlvh72zaaoxhxdajauozve5npi5kwryv4sj2ixayqhu4zgdd2nu' │ +│ 2 │ 'helia - ../gc/src' │ '502.72' │ '1.99' │ 503 │ '3.66' │ 'bafybeihhyvzl4zqbvvtafd6cnp37gwvrypn2cxpyr2yj5zppvgk3urxgpm' │ +│ 3 │ 'ipfs - src' │ '13.12' │ '76.22' │ 14 │ '99.63' │ 'bafybeic7zx457hr3s2z7n3rnl3nckwjo3nhlpex7kmzxw7tzv3y5fbyki4' │ +│ 4 │ 'ipfs - dist' │ '2.84' │ '352.04' │ 5 │ '418.21' │ 'bafybeiapdyhejcw6sd7f5bayzpp5in3rx44lu3rbpvl2opjgc2msevfsoe' │ +│ 5 │ 'ipfs - ../gc/src' │ '5.39' │ '185.56' │ 6 │ '217.48' │ 'bafybeibdpig6o56rjems2twzgvog7ssatt5szrpnjgvtnws4i4bm5csvoa' │ +│ 6 │ 'kubo - src' │ '17.47' │ '57.23' │ 18 │ '208.40' │ 'bafybeic7zx457hr3s2z7n3rnl3nckwjo3nhlpex7kmzxw7tzv3y5fbyki4' │ +│ 7 │ 'kubo - dist' │ '3.04' │ '328.87' │ 5 │ '1038.24' │ 'bafybeiapdyhejcw6sd7f5bayzpp5in3rx44lu3rbpvl2opjgc2msevfsoe' │ +│ 8 │ 'kubo - ../gc/src' │ '12.88' │ '77.65' │ 13 │ '328.25' │ 'bafybeifqlusi6zeboi7mxdbbjr5y5pdojrohhtelm4rbhb2vfkfa6f2kfu' │ +└─────────┴─────────────────────┴──────────┴──────────┴──────┴───────────┴───────────────────────────────────────────────────────────────┘ +``` + +### TEST_PATH +Test different paths + +```bash +TEST_PATH=../../node_modules/neo-async npm start + +> benchmarks-add-dir@1.0.0 start +> npm run build && node dist/src/index.js + + +> benchmarks-add-dir@1.0.0 build +> aegir build --bundle false + +[13:31:27] tsc [started] +[13:31:28] tsc [completed] +generating Ed25519 keypair... +(node:34722) ExperimentalWarning: The Fetch API is an experimental feature. This feature could change at any time +(Use `node --trace-warnings ...` to show where the warning was created) +┌─────────┬──────────────────────────────────────────────┬─────────┬───────────┬──────┬───────────┬───────────────────────────────────────────────────────────────┐ +│ (index) │ Implementation │ ops/s │ ms/op │ runs │ p99 │ CID │ +├─────────┼──────────────────────────────────────────────┼─────────┼───────────┼──────┼───────────┼───────────────────────────────────────────────────────────────┤ +│ 0 │ 'helia-fs - ../../node_modules/neo-async' │ '2.01' │ '498.48' │ 5 │ '2190.31' │ 'bafybeib5nofkubfon4upbeqvtn224uajsauqlkvlrik5p4xo53ws7e24sm' │ +│ 1 │ 'helia-mem - ../../node_modules/neo-async' │ '19.22' │ '52.04' │ 5 │ '85.31' │ 'bafybeib5nofkubfon4upbeqvtn224uajsauqlkvlrik5p4xo53ws7e24sm' │ +│ 2 │ 'ipfs - ../../node_modules/neo-async' │ '0.20' │ '4895.68' │ 5 │ '5209.99' │ 'bafybeigdyetiosfdnzg4cocoqneudndktcukaa3qdwj2ndeoxuqk6oxycm' │ +│ 3 │ 'kubo - ../../node_modules/neo-async' │ '0.38' │ '2641.59' │ 5 │ '7776.20' │ 'bafybeiey5wqhualgsssqo53dafzlp5fq2dzlv742raqvayougzsvbqvatm' │ +│ 4 │ 'kubo-direct - ../../node_modules/neo-async' │ '0.43' │ '2348.51' │ 5 │ '7149.77' │ 'bafybeiey5wqhualgsssqo53dafzlp5fq2dzlv742raqvayougzsvbqvatm' │ +└─────────┴──────────────────────────────────────────────┴─────────┴───────────┴──────┴───────────┴───────────────────────────────────────────────────────────────┘ +``` + +```bash +TEST_PATH=../../node_modules/ipfs-core/node_modules npm start + +> benchmarks-add-dir@1.0.0 start +> npm run build && node dist/src/index.js + + +> benchmarks-add-dir@1.0.0 build +> aegir build --bundle false + +[14:17:29] tsc [started] +[14:17:30] tsc [completed] +generating Ed25519 keypair... +(node:65964) ExperimentalWarning: The Fetch API is an experimental feature. This feature could change at any time +(Use `node --trace-warnings ...` to show where the warning was created) +generating Ed25519 keypair... +generating Ed25519 keypair... +┌─────────┬───────────────────────────────────────────────────────────┬────────┬─────────────┬──────┬─────────────┬───────────────────────────────────────────────────────────────┐ +│ (index) │ Implementation │ ops/s │ ms/op │ runs │ p99 │ CID │ +├─────────┼───────────────────────────────────────────────────────────┼────────┼─────────────┼──────┼─────────────┼───────────────────────────────────────────────────────────────┤ +│ 0 │ 'helia - ../../node_modules/ipfs-core/node_modules' │ '0.15' │ '6708.96' │ 5 │ '29599.11' │ 'bafybeihjqqav7quarfmhnejijq7edikz7rvryhuocpug5l7ovhvhvxjtwi' │ +│ 1 │ 'ipfs - ../../node_modules/ipfs-core/node_modules' │ '0.00' │ '228866.62' │ 5 │ '237419.96' │ 'bafybeicwkwides7xtqvxtc56vbmolrfli2ds2i3dmevghlibhmxmebir7u' │ +│ 2 │ 'kubo - ../../node_modules/ipfs-core/node_modules' │ '0.00' │ '230310.82' │ 5 │ '234432.20' │ 'bafybeicwkwides7xtqvxtc56vbmolrfli2ds2i3dmevghlibhmxmebir7u' │ +│ 3 │ 'kubo-direct - ../../node_modules/ipfs-core/node_modules' │ '0.00' │ '205561.86' │ 5 │ '219400.15' │ 'bafybeicwkwides7xtqvxtc56vbmolrfli2ds2i3dmevghlibhmxmebir7u' │ +└─────────┴───────────────────────────────────────────────────────────┴────────┴─────────────┴──────┴─────────────┴───────────────────────────────────────────────────────────────┘ +``` + +## Troubleshooting + +You can enable debug logging by setting `DEBUG=bench:add-dir*` diff --git a/benchmarks/add-dir/package.json b/benchmarks/add-dir/package.json new file mode 100644 index 00000000..85bd519a --- /dev/null +++ b/benchmarks/add-dir/package.json @@ -0,0 +1,37 @@ +{ + "name": "benchmarks-add-dir", + "version": "1.0.0", + "main": "index.js", + "private": true, + "type": "module", + "scripts": { + "clean": "aegir clean", + "build": "aegir build --bundle false", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "start": "npm run build && node dist/src/index.js" + }, + "devDependencies": { + "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-yamux": "^3.0.5", + "@helia/unixfs": "^1.4.0", + "@ipld/dag-pb": "^4.0.2", + "@libp2p/websockets": "^5.0.3", + "aegir": "^39.0.4", + "blockstore-fs": "^1.0.1", + "datastore-level": "^10.0.1", + "execa": "^7.0.0", + "go-ipfs": "^0.19.0", + "helia": "^1.0.0", + "ipfs-core": "^0.18.0", + "ipfs-unixfs-importer": "^15.1.5", + "ipfsd-ctl": "^13.0.0", + "it-all": "^2.0.0", + "it-drain": "^2.0.0", + "it-map": "^2.0.1", + "kubo-rpc-client": "^3.0.1", + "libp2p": "^0.43.0", + "multiformats": "^11.0.1", + "tinybench": "^2.4.0" + } +} diff --git a/benchmarks/add-dir/src/helia.ts b/benchmarks/add-dir/src/helia.ts new file mode 100644 index 00000000..a6db4f53 --- /dev/null +++ b/benchmarks/add-dir/src/helia.ts @@ -0,0 +1,78 @@ +import fs, { promises as fsPromises } from 'node:fs' +import os from 'node:os' +import nodePath from 'node:path' +import { type AddOptions, unixfs, globSource } from '@helia/unixfs' +import * as dagPb from '@ipld/dag-pb' +import { MemoryBlockstore } from 'blockstore-core' +import { FsBlockstore } from 'blockstore-fs' +import { MemoryDatastore } from 'datastore-core' +import { LevelDatastore } from 'datastore-level' +import { createHelia, type DAGWalker } from 'helia' +import { fixedSize } from 'ipfs-unixfs-importer/chunker' +import { balanced } from 'ipfs-unixfs-importer/layout' +import last from 'it-last' +import type { AddDirBenchmark } from './index.js' +import type { CID } from 'multiformats/cid' + +const dagPbWalker: DAGWalker = { + codec: dagPb.code, + async * walk (block) { + const node = dagPb.decode(block) + + yield * node.Links.map(l => l.Hash) + } +} + +const unixFsAddOptions: Partial = { + // default kubo options + cidVersion: 0, + rawLeaves: false, + layout: balanced({ + maxChildrenPerNode: 174 + }), + chunker: fixedSize({ + chunkSize: 262144 + }) +} +interface HeliaBenchmarkOptions { + blockstoreType?: 'fs' | 'mem' + datastoreType?: 'fs' | 'mem' +} + +export async function createHeliaBenchmark ({ blockstoreType = 'fs', datastoreType = 'fs' }: HeliaBenchmarkOptions = {}): Promise { + const repoPath = nodePath.join(os.tmpdir(), `helia-${Math.random()}`) + + const helia = await createHelia({ + blockstore: blockstoreType === 'fs' ? new FsBlockstore(`${repoPath}/blocks`) : new MemoryBlockstore(), + datastore: datastoreType === 'fs' ? new LevelDatastore(`${repoPath}/data`) : new MemoryDatastore(), + dagWalkers: [ + dagPbWalker + ], + start: false + }) + const unixFs = unixfs(helia) + + const addFile = async (path: string): Promise => unixFs.addFile({ + path: nodePath.relative(process.cwd(), path), + content: fs.createReadStream(path) + }, unixFsAddOptions) + + const addDir = async function (dir: string): Promise { + const res = await last(unixFs.addAll(globSource(nodePath.dirname(dir), `${nodePath.basename(dir)}/**/*`), unixFsAddOptions)) + + if (res == null) { + throw new Error('Import failed') + } + + return res.cid + } + + return { + async teardown () { + await helia.stop() + await fsPromises.rm(repoPath, { recursive: true, force: true }) + }, + addFile, + addDir + } +} diff --git a/benchmarks/add-dir/src/index.ts b/benchmarks/add-dir/src/index.ts new file mode 100644 index 00000000..2d4f27e5 --- /dev/null +++ b/benchmarks/add-dir/src/index.ts @@ -0,0 +1,182 @@ +/* eslint-disable no-console,no-loop-func */ + +import nodePath from 'node:path' +import debug from 'debug' +import { CID } from 'multiformats/cid' +import { Bench } from 'tinybench' +import { createHeliaBenchmark } from './helia.js' +import { createIpfsBenchmark } from './ipfs.js' +import { createKuboDirectBenchmark } from './kubo-direct.js' +import { createKuboBenchmark } from './kubo.js' + +const log = debug('bench:add-dir') +const ITERATIONS = parseInt(process.env.ITERATIONS ?? '5') +const MIN_TIME = parseInt(process.env.MIN_TIME ?? '1') +const TEST_PATH = process.env.TEST_PATH +const RESULT_PRECISION = 2 + +export interface AddDirBenchmark { + teardown: () => Promise + addFile?: (path: string) => Promise + addDir: (path: string) => Promise + getSize?: (cid: CID) => Promise +} + +interface BenchmarkTaskResult { + timing: number[] + cids: Map> + sizes: Map> +} + +const getDefaultResults = (): BenchmarkTaskResult => ({ + timing: [], + cids: new Map>(), + sizes: new Map>() +}) + +const impls: Array<{ name: string, create: () => Promise, results: BenchmarkTaskResult }> = [ + { + name: 'helia-fs', + create: async () => createHeliaBenchmark(), + results: getDefaultResults() + }, + { + name: 'helia-mem', + create: async () => createHeliaBenchmark({ blockstoreType: 'mem', datastoreType: 'mem' }), + results: getDefaultResults() + }, + { + name: 'ipfs', + create: async () => createIpfsBenchmark(), + results: getDefaultResults() + }, + { + name: 'kubo', + create: async () => createKuboBenchmark(), + results: getDefaultResults() + }, + { + name: 'kubo-direct', + create: async () => createKuboDirectBenchmark(), + results: getDefaultResults() + } +] + +async function main (): Promise { + let subject: AddDirBenchmark + + const suite = new Bench({ + iterations: ITERATIONS, + time: MIN_TIME, + setup: async (task) => { + log('Start: setup') + const impl = impls.find(({ name }) => task.name.includes(name)) + if (impl != null) { + subject = await impl.create() + } else { + throw new Error(`No implementation with name '${task.name}'`) + } + log('End: setup') + }, + teardown: async () => { + log('Start: teardown') + await subject.teardown() + log('End: teardown') + } + }) + + const testPaths = TEST_PATH != null + ? [TEST_PATH] + : [ + nodePath.relative(process.cwd(), nodePath.join(process.cwd(), 'src')), + nodePath.relative(process.cwd(), nodePath.join(process.cwd(), 'dist')), + nodePath.relative(process.cwd(), nodePath.join(process.cwd(), '..', 'gc', 'src')) + ] + + for (const impl of impls) { + for (const testPath of testPaths) { + const absPath = nodePath.join(process.cwd(), testPath) + suite.add(`${impl.name} - ${testPath}`, async function () { + const start = Date.now() + const cid = await subject.addDir(absPath) + impl.results.timing.push(Date.now() - start) + const cidSet = impl.results.cids.get(testPath) ?? new Set() + cidSet.add(cid.toString()) + impl.results.cids.set(testPath, cidSet) + }, + { + beforeEach: async () => { + log(`Start: test ${impl.name}`) + }, + afterEach: async () => { + log(`End: test ${impl.name}`) + const cidSet = impl.results.cids.get(testPath) + if (cidSet != null) { + for (const cid of cidSet.values()) { + const size = await subject.getSize?.(CID.parse(cid)) + if (size != null) { + const statsSet = impl.results.sizes.get(testPath) ?? new Set() + statsSet.add(size?.toString()) + impl.results.sizes.set(testPath, statsSet) + } + } + } + } + } + ) + } + } + + await suite.run() + + if (process.env.INCREMENT != null) { + if (process.env.ITERATION === '1') { + console.info('implementation, count, add dir (ms), cid') + } + + for (const impl of impls) { + console.info( + `${impl.name},`, + `${process.env.INCREMENT},`, + `${(impl.results.timing.reduce((acc, curr) => acc + curr, 0) / impl.results.timing.length).toFixed(RESULT_PRECISION)},` + ) + } + } else { + const implCids: Record = {} + const implSizes: Record = {} + for (const impl of impls) { + for (const [testPath, cids] of impl.results.cids.entries()) { + implCids[`${impl.name} - ${testPath}`] = Array.from(cids).join(', ') + } + for (const [testPath, sizes] of impl.results.sizes.entries()) { + implSizes[`${impl.name} - ${testPath}`] = Array.from(sizes).join(', ') + } + } + console.table(suite.tasks.map(({ name, result }) => { + if (result?.error != null) { + return { + Implementation: name, + 'ops/s': 'error', + 'ms/op': 'error', + runs: 'error', + p99: 'error', + CID: (result?.error as any)?.message + } + } + return { + Implementation: name, + 'ops/s': result?.hz.toFixed(RESULT_PRECISION), + 'ms/op': result?.period.toFixed(RESULT_PRECISION), + runs: result?.samples.length, + p99: result?.p99.toFixed(RESULT_PRECISION), + CID: implCids[name] + } + })) + } + process.exit(0) // sometimes the test hangs (need to debug) +} + +main().catch(err => { + console.error(err) // eslint-disable-line no-console + process.exit(1) +}) diff --git a/benchmarks/add-dir/src/ipfs.ts b/benchmarks/add-dir/src/ipfs.ts new file mode 100644 index 00000000..926094a6 --- /dev/null +++ b/benchmarks/add-dir/src/ipfs.ts @@ -0,0 +1,46 @@ +import fs, { promises as fsPromises } from 'node:fs' +import os from 'node:os' +import nodePath from 'node:path' +import { create, globSource } from 'ipfs-core' +import last from 'it-last' +import type { AddDirBenchmark } from './index.js' +import type { CID } from 'multiformats/cid' + +export async function createIpfsBenchmark (): Promise { + const repoPath = nodePath.join(os.tmpdir(), `ipfs-${Math.random()}`) + + const ipfs = await create({ + config: { + Addresses: { + Swarm: [] + } + }, + repo: repoPath, + start: false, + init: { + emptyRepo: true + } + }) + + const addFile = async (path: string): Promise => (await ipfs.add({ path: nodePath.relative(process.cwd(), path), content: fs.createReadStream(path) }, { cidVersion: 1, pin: false })).cid + + const addDir = async function (dir: string): Promise { + // @ts-expect-error types are messed up + const res = await last(ipfs.addAll(globSource(nodePath.dirname(dir), `${nodePath.basename(dir)}/**/*`))) + + if (res == null) { + throw new Error('Import failed') + } + + return res.cid + } + + return { + async teardown () { + await ipfs.stop() + await fsPromises.rm(repoPath, { recursive: true, force: true }) + }, + addFile, + addDir + } +} diff --git a/benchmarks/add-dir/src/kubo-direct.ts b/benchmarks/add-dir/src/kubo-direct.ts new file mode 100644 index 00000000..bb0ae242 --- /dev/null +++ b/benchmarks/add-dir/src/kubo-direct.ts @@ -0,0 +1,30 @@ +import { promises as fsPromises } from 'node:fs' +import os from 'node:os' +import nodePath from 'node:path' +import { execa } from 'execa' +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { CID } from 'multiformats/cid' +import type { AddDirBenchmark } from './index.js' + +export async function createKuboDirectBenchmark (): Promise { + const repoDir = nodePath.join(os.tmpdir(), 'kubo-direct') + + await execa(goIpfs.path(), ['--repo-dir', repoDir, 'init']) + + const addDir = async function (dir: string): Promise { + const { stdout } = await execa(goIpfs.path(), ['--repo-dir', repoDir, 'add', '-r', '--pin=false', dir]) + const lines = stdout.split('\n') + const lastLine = lines.pop() + const cid = CID.parse(lastLine?.split(' ')[1] as string) + + return cid + } + + return { + async teardown () { + await fsPromises.rm(repoDir, { recursive: true, force: true }) + }, + addDir + } +} diff --git a/benchmarks/add-dir/src/kubo.ts b/benchmarks/add-dir/src/kubo.ts new file mode 100644 index 00000000..e0d8552a --- /dev/null +++ b/benchmarks/add-dir/src/kubo.ts @@ -0,0 +1,46 @@ +import fs, { promises as fsPromises } from 'node:fs' +import nodePath from 'node:path' +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { createController } from 'ipfsd-ctl' +import last from 'it-last' +import * as goRpcClient from 'kubo-rpc-client' +import type { AddDirBenchmark } from './index.js' +import type { CID } from 'multiformats/cid' + +export async function createKuboBenchmark (): Promise { + const controller = await createController({ + type: 'go', + test: true, + ipfsBin: goIpfs.path(), + kuboRpcModule: goRpcClient, + ipfsOptions: { + init: { + emptyRepo: true + } + } + }) + + const addFile = async (path: string): Promise => (await controller.api.add({ path: nodePath.relative(process.cwd(), path), content: fs.createReadStream(path) }, { cidVersion: 1, pin: false })).cid + + const addDir = async function (dir: string): Promise { + // @ts-expect-error types are messed up + const res = await last(controller.api.addAll(goRpcClient.globSource(nodePath.dirname(dir), `${nodePath.basename(dir)}/**/*`))) + + if (res == null) { + throw new Error('Import failed') + } + + return res.cid + } + + return { + async teardown () { + const { repoPath } = await controller.api.repo.stat() + await controller.stop() + await fsPromises.rm(repoPath, { recursive: true, force: true }) + }, + addFile, + addDir + } +} diff --git a/benchmarks/add-dir/tsconfig.json b/benchmarks/add-dir/tsconfig.json new file mode 100644 index 00000000..a9160320 --- /dev/null +++ b/benchmarks/add-dir/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist", + "target": "ES2022", + "module": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"] + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../../packages/helia" + } + ] +}