Merge pull request #1611 from contentstack/feat/config-rate-limit

feat/config rate limit
contentstack · Oct 9, 2024 · f5b5c37 · f5b5c37
2 parents 43bdc8a + 8d3927a
commit f5b5c37
Show file tree

Hide file tree

Showing 23 changed files with 1,248 additions and 643 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/contentstack-branches/package.json b/packages/contentstack-branches/package.json
@@ -26,7 +26,7 @@
   },
   "devDependencies": {
     "@contentstack/cli-auth": "~1.3.22",
-    "@contentstack/cli-config": "~1.7.3",
+    "@contentstack/cli-config": "~1.8.0",
     "@contentstack/cli-dev-dependencies": "~1.2.4",
     "@oclif/plugin-help": "^5.1.19",
     "@oclif/test": "^2.5.6",

diff --git a/packages/contentstack-bulk-publish/README.md b/packages/contentstack-bulk-publish/README.md
@@ -18,7 +18,7 @@ $ npm install -g @contentstack/cli-cm-bulk-publish
 $ csdx COMMAND
 running command...
 $ csdx (--version)
-@contentstack/cli-cm-bulk-publish/1.4.9 darwin-arm64 node-v22.2.0
+@contentstack/cli-cm-bulk-publish/1.5.0 darwin-arm64 node-v22.2.0
 $ csdx --help [COMMAND]
 USAGE
   $ csdx COMMAND

diff --git a/packages/contentstack-bulk-publish/package.json b/packages/contentstack-bulk-publish/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@contentstack/cli-cm-bulk-publish",
   "description": "Contentstack CLI plugin for bulk publish actions",
-  "version": "1.4.9",
+  "version": "1.5.0",
   "author": "Contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {

diff --git a/packages/contentstack-bulk-publish/src/producer/publish-assets.js b/packages/contentstack-bulk-publish/src/producer/publish-assets.js
@@ -6,6 +6,7 @@ const { performBulkPublish, publishAsset, initializeLogger } = require('../consu
 const retryFailedLogs = require('../util/retryfailed');
 const { validateFile } = require('../util/fs');
 const { isEmpty } = require('../util');
+const { fetchBulkPublishLimit } = require('../util/common-utility');
 
 const queue = getQueue();
 let logFileName;
@@ -14,7 +15,7 @@ let filePath;
 
 /* eslint-disable no-param-reassign */
 
-async function getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, skip = 0) {
+async function getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, bulkPublishLimit, skip = 0) {
   return new Promise((resolve, reject) => {
     let queryParams = {
       folder: folder,
@@ -33,18 +34,27 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe
           let assets = assetResponse.items;
           for (let index = 0; index < assetResponse.items.length; index++) {
             if (assets[index].is_dir === true) {
-              await getAssets(stack, assets[index].uid, bulkPublish, environments, locale, apiVersion, 0);
+              await getAssets(
+                stack,
+                assets[index].uid,
+                bulkPublish,
+                environments,
+                locale,
+                apiVersion,
+                bulkPublishLimit,
+                0,
+              );
               continue;
             }
             if (bulkPublish) {
-              if (bulkPublishSet.length < 10) {
+              if (bulkPublishSet.length < bulkPublishLimit) {
                 bulkPublishSet.push({
                   uid: assets[index].uid,
                   locale,
                   publish_details: assets[index].publish_details || [],
                 });
               }
-              if (bulkPublishSet.length === 10) {
+              if (bulkPublishSet.length === bulkPublishLimit) {
                 await queue.Enqueue({
                   assets: bulkPublishSet,
                   Type: 'asset',
@@ -56,7 +66,11 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe
                 bulkPublishSet = [];
               }
 
-              if (assetResponse.items.length - 1 === index && bulkPublishSet.length > 0 && bulkPublishSet.length < 10) {
+              if (
+                assetResponse.items.length - 1 === index &&
+                bulkPublishSet.length > 0 &&
+                bulkPublishSet.length < bulkPublishLimit
+              ) {
                 await queue.Enqueue({
                   assets: bulkPublishSet,
                   Type: 'asset',
@@ -81,7 +95,7 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe
           if (skip === assetResponse.count) {
             return resolve(true);
           }
-          await getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, skip);
+          await getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, bulkPublishLimit, skip);
           return resolve();
         } else {
           resolve();
@@ -133,8 +147,9 @@ async function start({ retryFailed, bulkPublish, environments, folderUid, locale
     }
   } else if (folderUid) {
     setConfig(config, bulkPublish);
+    const bulkPublishLimit = fetchBulkPublishLimit(stack?.org_uid);
     for (const element of locales) {
-      await getAssets(stack, folderUid, bulkPublish, environments, element, apiVersion);
+      await getAssets(stack, folderUid, bulkPublish, environments, element, apiVersion, bulkPublishLimit);
     }
   }
 }