show the progressbar while downloading spec tests

package.json

@@ -89,5 +89,8 @@
     "loupe": "^2.3.6",
     "vite": "^5.3.4",
     "testcontainers/**/nan": "^2.19.0"
+  },
+  "dependencies": {
+    "progress-stream": "^2.0.0"
   }
 }

packages/beacon-node/package.json

@@ -133,6 +133,7 @@
     "@lodestar/utils": "^1.23.0",
     "@lodestar/validator": "^1.23.0",
     "@multiformats/multiaddr": "^12.1.3",
+    "axios": "^1.7.7",
     "c-kzg": "^2.1.2",
     "datastore-core": "^9.1.1",
     "datastore-level": "^10.1.1",
@@ -144,16 +145,19 @@
     "jwt-simple": "0.5.6",
     "libp2p": "1.4.3",
     "multiformats": "^11.0.1",
+    "progress-stream": "^2.0.0",
     "prom-client": "^15.1.0",
     "qs": "^6.11.1",
     "snappyjs": "^0.7.0",
     "strict-event-emitter-types": "^2.0.0",
     "systeminformation": "^5.22.9",
+    "tar": "^7.4.3",
     "uint8arraylist": "^2.4.7",
     "xxhash-wasm": "1.0.2"
   },
   "devDependencies": {
     "@types/datastore-level": "^3.0.0",
+    "@types/progress-stream": "^2.0.5",
     "@types/qs": "^6.9.7",
     "@types/tmp": "^0.2.3",
     "it-drain": "^3.0.3",

packages/spec-test-util/src/downloadTests.ts

@@ -4,6 +4,7 @@ import path from "node:path";
 import stream from "node:stream";
 import {promisify} from "node:util";
 import {retry} from "@lodestar/utils";
+import progressStream from "progress-stream";
 import axios from "axios";
 import {rimraf} from "rimraf";
 import {x as extractTar} from "tar";
@@ -15,9 +16,9 @@ const logEmpty = (): void => {};
 export type DownloadTestsOptions = {
   specVersion: string;
   outputDir: string;
-  /** Root Github URL `https://github.com/ethereum/consensus-spec-tests` */
+    /** Root Github URL `https://github.com/ethereum/consensus-spec-tests` */
   specTestsRepoUrl: string;
-  /** Release files names to download without prefix `["general", "mainnet", "minimal"]` */
+    /** Release files names to download without prefix `["general", "mainnet", "minimal"]` */
   testsToDownload: string[];
 };
 
@@ -74,13 +75,29 @@ export async function downloadGenericSpecTests<TestNames extends string>(
             timeout: 30 * 60 * 1000,
           });
 
-          const totalSize = headers["content-length"] as string;
+          const totalSize = parseInt(headers["content-length"], 10);
           log(`Downloading ${url} - ${totalSize} bytes`);
 
-          // extract tar into output directory
-          await promisify(stream.pipeline)(data, extractTar({cwd: outputDir}));
+          const progress = progressStream({ length: totalSize, time: 100 });
+          const barLength = 40; // Length of the progress bar
+          let lastPercentage = 0;
+
+          progress.on('progress', (progressData) => {
+            const currentPercentage = Math.floor(progressData.percentage);
+
+            if (currentPercentage > lastPercentage) {
+              lastPercentage = currentPercentage;
+              const filledLength = Math.round(barLength * currentPercentage / 100);
+              const bar = '#'.repeat(filledLength) + '-'.repeat(barLength - filledLength);
+              // Move the cursor to the beginning of the line and overwrite it
+              process.stdout.write(`\r[${bar}] ${currentPercentage}% (${progressData.transferred} bytes)`);
+            }
+          });
 
-          log(`Downloaded  ${url}`);
+          // Use pipeline to handle the stream and extract the tar
+          await promisify(stream.pipeline)(data.pipe(progress), extractTar({ cwd: outputDir }));
+          console.log(); // Move to the next line after the download is complete
+          log(`Downloaded ${url}`);
         },
         {
           retries: 3,

packages/types/src/progress-stream.d.ts

@@ -0,0 +1,13 @@
+declare module 'progress-stream' {
+    import { Transform } from 'stream';
+
+    interface ProgressStreamOptions {
+      length: number; // Total length of the data being processed
+      time?: number;  // Time interval to emit progress events
+    }
+
+    function progressStream(options?: ProgressStreamOptions): Transform;
+
+    export = progressStream;
+  }
+

tsconfig.json

@@ -2,6 +2,8 @@
   "extends": "./tsconfig.build.json",
   "compilerOptions": {
     "emitDeclarationOnly": false,
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
     "incremental": false,
     // Required to run benchmark command from root directory
     "typeRoots": ["node_modules/@types", "./types"],

yarn.lock

@@ -2979,6 +2979,13 @@
   resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.7.0.tgz#0868518771004753acb07d9871cf5b143b51dc30"
   integrity sha512-cEJ2ebtV1v/5Ilb55E05J6F5SrHKQWzUttIhR5Mkayyo+yvPslcpByuFC3D+J7X1ebziTOBpWuMpUdjLfh3SMQ==
 
+"@types/progress-stream@^2.0.5":
+  version "2.0.5"
+  resolved "https://registry.npmjs.org/@types/progress-stream/-/progress-stream-2.0.5.tgz"
+  integrity sha512-5YNriuEZkHlFHHepLIaxzq3atGeav1qCTGzB74HKWpo66qjfostF+rHc785YYYHeBytve8ZG3ejg42jEIfXNiQ==
+  dependencies:
+    "@types/node" "*"
+
 "@swc/[email protected]":
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.7.0.tgz#43048d67f871e798566c3883c991bed521ef3b9e"
@@ -10185,6 +10192,19 @@ pathval@^2.0.0:
   resolved "https://registry.yarnpkg.com/pathval/-/pathval-2.0.0.tgz#7e2550b422601d4f6b8e26f1301bc8f15a741a25"
   integrity sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==
 
+progress-stream@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.npmjs.org/progress-stream/-/progress-stream-2.0.0.tgz"
+  integrity sha512-xJwOWR46jcXUq6EH9yYyqp+I52skPySOeHfkxOZ2IY1AiBi/sFJhbhAKHoV3OTw/omQ45KTio9215dRJ2Yxd3Q==
+  dependencies:
+    speedometer "~1.0.0"
+    through2 "~2.0.3"
+
+progress@^2.0.3, [email protected]:
+  version "2.0.3"
+  resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz"
+  integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
+
 pbkdf2@^3.0.3:
   version "3.1.2"
   resolved "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz"