From a0730205877de905d4f81eefd165d15b8aef0b7c Mon Sep 17 00:00:00 2001 From: mihirrchauhan Date: Tue, 26 Nov 2024 11:20:02 +0530 Subject: [PATCH] show the progressbar while downloading spec tests --- package.json | 3 ++ packages/beacon-node/package.json | 4 +++ packages/spec-test-util/src/downloadTests.ts | 29 ++++++++++++++++---- packages/types/src/progress-stream.d.ts | 13 +++++++++ tsconfig.json | 2 ++ yarn.lock | 20 ++++++++++++++ 6 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 packages/types/src/progress-stream.d.ts diff --git a/package.json b/package.json index e6de076cf494..d9b9676799a1 100644 --- a/package.json +++ b/package.json @@ -89,5 +89,8 @@ "loupe": "^2.3.6", "vite": "^5.3.4", "testcontainers/**/nan": "^2.19.0" + }, + "dependencies": { + "progress-stream": "^2.0.0" } } diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 3213504334e3..2e58e62f8ef7 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -132,6 +132,7 @@ "@lodestar/utils": "^1.22.0", "@lodestar/validator": "^1.22.0", "@multiformats/multiaddr": "^12.1.3", + "axios": "^1.7.7", "c-kzg": "^2.1.2", "datastore-core": "^10.0.0", "datastore-level": "^11.0.0", @@ -143,16 +144,19 @@ "jwt-simple": "0.5.6", "libp2p": "2.1.7", "multiformats": "^11.0.1", + "progress-stream": "^2.0.0", "prom-client": "^15.1.0", "qs": "^6.11.1", "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.22.9", + "tar": "^7.4.3", "uint8arraylist": "^2.4.7", "xxhash-wasm": "1.0.2" }, "devDependencies": { "@types/datastore-level": "^3.0.0", + "@types/progress-stream": "^2.0.5", "@types/qs": "^6.9.7", "@types/tmp": "^0.2.3", "it-drain": "^3.0.3", diff --git a/packages/spec-test-util/src/downloadTests.ts b/packages/spec-test-util/src/downloadTests.ts index bca6a8ac52c9..926778d26860 100644 --- a/packages/spec-test-util/src/downloadTests.ts +++ b/packages/spec-test-util/src/downloadTests.ts @@ -7,6 +7,7 @@ import {rimraf} from "rimraf"; import axios from "axios"; import {x as extractTar} from "tar"; import {retry} from "@lodestar/utils"; +import progressStream from "progress-stream"; export const defaultSpecTestsRepoUrl = "https://github.com/ethereum/consensus-spec-tests"; @@ -15,9 +16,9 @@ const logEmpty = (): void => {}; export type DownloadTestsOptions = { specVersion: string; outputDir: string; - /** Root Github URL `https://github.com/ethereum/consensus-spec-tests` */ + /** Root Github URL `https://github.com/ethereum/consensus-spec-tests` */ specTestsRepoUrl: string; - /** Release files names to download without prefix `["general", "mainnet", "minimal"]` */ + /** Release files names to download without prefix `["general", "mainnet", "minimal"]` */ testsToDownload: string[]; }; @@ -74,13 +75,29 @@ export async function downloadGenericSpecTests( timeout: 30 * 60 * 1000, }); - const totalSize = headers["content-length"] as string; + const totalSize = parseInt(headers["content-length"], 10); log(`Downloading ${url} - ${totalSize} bytes`); - // extract tar into output directory - await promisify(stream.pipeline)(data, extractTar({cwd: outputDir})); + const progress = progressStream({ length: totalSize, time: 100 }); + const barLength = 40; // Length of the progress bar + let lastPercentage = 0; + + progress.on('progress', (progressData) => { + const currentPercentage = Math.floor(progressData.percentage); + + if (currentPercentage > lastPercentage) { + lastPercentage = currentPercentage; + const filledLength = Math.round(barLength * currentPercentage / 100); + const bar = '#'.repeat(filledLength) + '-'.repeat(barLength - filledLength); + // Move the cursor to the beginning of the line and overwrite it + process.stdout.write(`\r[${bar}] ${currentPercentage}% (${progressData.transferred} bytes)`); + } + }); - log(`Downloaded ${url}`); + // Use pipeline to handle the stream and extract the tar + await promisify(stream.pipeline)(data.pipe(progress), extractTar({ cwd: outputDir })); + console.log(); // Move to the next line after the download is complete + log(`Downloaded ${url}`); }, { retries: 3, diff --git a/packages/types/src/progress-stream.d.ts b/packages/types/src/progress-stream.d.ts new file mode 100644 index 000000000000..c1c7c76a096a --- /dev/null +++ b/packages/types/src/progress-stream.d.ts @@ -0,0 +1,13 @@ +declare module 'progress-stream' { + import { Transform } from 'stream'; + + interface ProgressStreamOptions { + length: number; // Total length of the data being processed + time?: number; // Time interval to emit progress events + } + + function progressStream(options?: ProgressStreamOptions): Transform; + + export = progressStream; + } + \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index c2cf3e5f258a..0dbd3ac9baed 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,6 +2,8 @@ "extends": "./tsconfig.build.json", "compilerOptions": { "emitDeclarationOnly": false, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, "incremental": false, // Required to run benchmark command from root directory "typeRoots": ["node_modules/@types", "./types"], diff --git a/yarn.lock b/yarn.lock index 8519ed6434bf..d76748c19ec9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3003,6 +3003,13 @@ resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.7.0.tgz#0868518771004753acb07d9871cf5b143b51dc30" integrity sha512-cEJ2ebtV1v/5Ilb55E05J6F5SrHKQWzUttIhR5Mkayyo+yvPslcpByuFC3D+J7X1ebziTOBpWuMpUdjLfh3SMQ== +"@types/progress-stream@^2.0.5": + version "2.0.5" + resolved "https://registry.npmjs.org/@types/progress-stream/-/progress-stream-2.0.5.tgz" + integrity sha512-5YNriuEZkHlFHHepLIaxzq3atGeav1qCTGzB74HKWpo66qjfostF+rHc785YYYHeBytve8ZG3ejg42jEIfXNiQ== + dependencies: + "@types/node" "*" + "@swc/core-win32-arm64-msvc@1.7.0": version "1.7.0" resolved "https://registry.yarnpkg.com/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.7.0.tgz#43048d67f871e798566c3883c991bed521ef3b9e" @@ -10304,6 +10311,19 @@ pathval@^2.0.0: resolved "https://registry.yarnpkg.com/pathval/-/pathval-2.0.0.tgz#7e2550b422601d4f6b8e26f1301bc8f15a741a25" integrity sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA== +progress-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/progress-stream/-/progress-stream-2.0.0.tgz" + integrity sha512-xJwOWR46jcXUq6EH9yYyqp+I52skPySOeHfkxOZ2IY1AiBi/sFJhbhAKHoV3OTw/omQ45KTio9215dRJ2Yxd3Q== + dependencies: + speedometer "~1.0.0" + through2 "~2.0.3" + +progress@^2.0.3, progress@2.0.3: + version "2.0.3" + resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz" + integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== + pbkdf2@^3.0.3: version "3.1.2" resolved "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz"