refactor: Refactor Upload Directory for Improved Reliability (#1867)

* build: Update engines from 8 to 10 Will update to 10 to 12 very soon, but this will prevent eslint issues today * refactor: Refactor Upload Directory for Improved Reliability * chore: Fix file naming * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: full Node 10 compatiblity * refactor: use defaults from sample comments * fix: Promise-related fixes * fix: add prefix items with dirname Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
googleapis · Apr 13, 2022 · 640f7b0 · 640f7b0
1 parent 5f78907
commit 640f7b0
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 55 deletions.
diff --git a/samples/package.json b/samples/package.json
@@ -4,7 +4,7 @@
   "license": "Apache-2.0",
   "author": "Google Inc.",
   "engines": {
-    "node": ">=8"
+    "node": ">=10"
   },
   "repository": "googleapis/nodejs-storage",
   "private": true,

diff --git a/samples/uploadDirectory.js b/samples/uploadDirectory.js
@@ -19,7 +19,10 @@
 //   description: Uploads full hierarchy of a local directory to a bucket.
 //   usage: node files.js upload-directory <bucketName> <directoryPath>
 
-async function main(bucketName, directoryPath) {
+function main(
+  bucketName = 'your-unique-bucket-name',
+  directoryPath = './local/path/to/directory'
+) {
   // [START upload_directory]
   /**
    * TODO(developer): Uncomment the following lines before running the sample.
@@ -36,68 +39,53 @@ async function main(bucketName, directoryPath) {
   // Creates a client
   const storage = new Storage();
 
+  const {promisify} = require('util');
   const fs = require('fs');
   const path = require('path');
-  const fileList = [];
 
-  async function uploadDirectory() {
-    // Get a list of files from the specified directory
-    let dirCtr = 1;
-    let itemCtr = 0;
-    const pathDirName = path.dirname(directoryPath);
-
-    getFiles(directoryPath);
-
-    function getFiles(directory) {
-      fs.readdir(directory, (err, items) => {
-        dirCtr--;
-        itemCtr += items.length;
-        items.forEach(item => {
-          const fullPath = path.join(directory, item);
-          fs.stat(fullPath, (err, stat) => {
-            itemCtr--;
-            if (stat.isFile()) {
-              fileList.push(fullPath);
-            } else if (stat.isDirectory()) {
-              dirCtr++;
-              getFiles(fullPath);
-            }
-            if (dirCtr === 0 && itemCtr === 0) {
-              onComplete();
-            }
-          });
-        });
-      });
+  const readdir = promisify(fs.readdir);
+  const stat = promisify(fs.stat);
+
+  async function* getFiles(directory = '.') {
+    for (const file of await readdir(directory)) {
+      const fullPath = path.join(directory, file);
+      const stats = await stat(fullPath);
+
+      if (stats.isDirectory()) {
+        yield* getFiles(fullPath);
+      }
+
+      if (stats.isFile()) {
+        yield fullPath;
+      }
     }
+  }
 
-    async function onComplete() {
-      const resp = await Promise.all(
-        fileList.map(filePath => {
-          let destination = path.relative(pathDirName, filePath);
-          // If running on Windows
-          if (process.platform === 'win32') {
-            destination = destination.replace(/\\/g, '/');
-          }
-          return storage
-            .bucket(bucketName)
-            .upload(filePath, {destination})
-            .then(
-              uploadResp => ({fileName: destination, status: uploadResp[0]}),
-              err => ({fileName: destination, response: err})
-            );
-        })
-      );
-
-      const successfulUploads =
-        fileList.length - resp.filter(r => r.status instanceof Error).length;
-      console.log(
-        `${successfulUploads} files uploaded to ${bucketName} successfully.`
-      );
+  async function uploadDirectory() {
+    const bucket = storage.bucket(bucketName);
+    let successfulUploads = 0;
+
+    for await (const filePath of getFiles(directoryPath)) {
+      try {
+        const dirname = path.dirname(directoryPath);
+        const destination = path.relative(dirname, filePath);
+
+        await bucket.upload(filePath, {destination});
+
+        console.log(`Successfully uploaded: ${filePath}`);
+        successfulUploads++;
+      } catch (e) {
+        console.error(`Error uploading ${filePath}:`, e);
+      }
     }
+
+    console.log(
+      `${successfulUploads} files uploaded to ${bucketName} successfully.`
+    );
   }
 
   uploadDirectory().catch(console.error);
   // [END upload_directory]
 }
 
-main(...process.argv.slice(2)).catch(console.error);
+main(...process.argv.slice(2));