Merge pull request #1171 from contentstack/release/hotfix

Release: Audit hotfix release
contentstack · Nov 23, 2023 · 6a36791 · 6a36791
2 parents c4e9c47 + 7aceb49
commit 6a36791
Show file tree

Hide file tree

Showing 64 changed files with 4,151 additions and 1,278 deletions.
diff --git a/.gitignore b/.gitignore
@@ -16,4 +16,6 @@ contentstack-cli-logs
 packages/**/package-lock.json
 .dccache
 yarn.lock
-contents-*
+contents-*
+*.http
+*.todo
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/contentstack-audit/README.md b/packages/contentstack-audit/README.md
@@ -56,16 +56,20 @@ USAGE
     --no-truncate]
 
 FLAGS
+  --modules=<option>...  Provide the list of modules to be audited
+                         <options: content-types|global-fields|entries>
+  --report-path=<value>  Path to store the audit reports
+
+COMMON FLAGS
   -c, --config=<value>    Path of the external config
   -d, --data-dir=<value>  Path where the data is stored
-  --columns=<value>       only show provided columns (comma-separated)
-  --csv                   output is csv format [alias: --output=csv]
-  --filter=<value>        filter property by partial string matching, ex: name=foo
-  --modules=<option>...   Provide the list of modules to be audited
-                          <options: content-types|global-fields|entries>
-  --no-truncate           do not truncate output to fit screen
-  --report-path=<value>   Path to store the audit reports
-  --sort=<value>          property to sort by (prepend '-' for descending)
+
+TABLE FLAGS
+  --columns=<value>  Show only the specified columns (comma-separated)
+  --csv              The output is in the CSV format [alias: --output=csv]
+  --filter=<value>   Filter property by partial string matching. For example: name=foo
+  --no-truncate      The output is not truncated to fit the screen
+  --sort=<value>     Property to sort by (prepend '-' for descending)
 
 DESCRIPTION
   Perform audits and find possible errors in the exported Contentstack data
@@ -93,22 +97,29 @@ Perform audits and fix possible errors in the exported Contentstack data.
 ```
 USAGE
   $ csdx audit:fix [-c <value>] [-d <value>] [--report-path <value>] [--modules
-    content-types|global-fields|entries] [--copy-path <value> --copy-dir] [--columns <value> | ] [--sort <value>]
-    [--filter <value>] [--csv | --no-truncate]
+    content-types|global-fields|entries] [--copy-path <value> --copy-dir] [--fix-only
+    reference|global_field|json:rte|json:custom-field|blocks|group] [--columns <value> | ] [--sort <value>] [--filter
+    <value>] [--csv | --no-truncate]
 
 FLAGS
-  -c, --config=<value>    Path of the external config
-  -d, --data-dir=<value>  Path where the data is stored
-  --columns=<value>       only show provided columns (comma-separated)
   --copy-dir              Create backup from the original data.
   --copy-path=<value>     Provide the path to backup the copied data
-  --csv                   output is csv format [alias: --output=csv]
-  --filter=<value>        filter property by partial string matching, ex: name=foo
+  --fix-only=<option>...  Provide the list of fix options
+                          <options: reference|global_field|json:rte|json:custom-field|blocks|group>
   --modules=<option>...   Provide the list of modules to be audited
                           <options: content-types|global-fields|entries>
-  --no-truncate           do not truncate output to fit screen
   --report-path=<value>   Path to store the audit reports
-  --sort=<value>          property to sort by (prepend '-' for descending)
+
+COMMON FLAGS
+  -c, --config=<value>    Path of the external config
+  -d, --data-dir=<value>  Path where the data is stored
+
+TABLE FLAGS
+  --columns=<value>  Show only the specified columns (comma-separated)
+  --csv              The output is in the CSV format [alias: --output=csv]
+  --filter=<value>   Filter property by partial string matching. For example: name=foo
+  --no-truncate      The output is not truncated to fit the screen
+  --sort=<value>     Property to sort by (prepend '-' for descending)
 
 DESCRIPTION
   Perform audits and fix possible errors in the exported Contentstack data.
@@ -124,6 +135,8 @@ EXAMPLES
 
   $ csdx audit:fix --report-path=<path> --copy-dir --csv
 
+  $ csdx audit:fix --fix-only=reference,global_field --copy-dir
+
   $ csdx audit:fix --report-path=<path> --filter="name=<filter-value>"
 
   $ csdx audit:fix --report-path=<path> --modules=content-types --filter="name="<filter-value>" --copy-dir --copy-path=<path>
@@ -140,16 +153,20 @@ USAGE
     --no-truncate]
 
 FLAGS
+  --modules=<option>...  Provide the list of modules to be audited
+                         <options: content-types|global-fields|entries>
+  --report-path=<value>  Path to store the audit reports
+
+COMMON FLAGS
   -c, --config=<value>    Path of the external config
   -d, --data-dir=<value>  Path where the data is stored
-  --columns=<value>       only show provided columns (comma-separated)
-  --csv                   output is csv format [alias: --output=csv]
-  --filter=<value>        filter property by partial string matching, ex: name=foo
-  --modules=<option>...   Provide the list of modules to be audited
-                          <options: content-types|global-fields|entries>
-  --no-truncate           do not truncate output to fit screen
-  --report-path=<value>   Path to store the audit reports
-  --sort=<value>          property to sort by (prepend '-' for descending)
+
+TABLE FLAGS
+  --columns=<value>  Show only the specified columns (comma-separated)
+  --csv              The output is in the CSV format [alias: --output=csv]
+  --filter=<value>   Filter property by partial string matching. For example: name=foo
+  --no-truncate      The output is not truncated to fit the screen
+  --sort=<value>     Property to sort by (prepend '-' for descending)
 
 DESCRIPTION
   Perform audits and find possible errors in the exported Contentstack data
@@ -179,22 +196,29 @@ Perform audits and fix possible errors in the exported Contentstack data.
 ```
 USAGE
   $ csdx cm:stacks:audit:fix [-c <value>] [-d <value>] [--report-path <value>] [--modules
-    content-types|global-fields|entries] [--copy-path <value> --copy-dir] [--columns <value> | ] [--sort <value>]
-    [--filter <value>] [--csv | --no-truncate]
+    content-types|global-fields|entries] [--copy-path <value> --copy-dir] [--fix-only
+    reference|global_field|json:rte|json:custom-field|blocks|group] [--columns <value> | ] [--sort <value>] [--filter
+    <value>] [--csv | --no-truncate]
 
 FLAGS
-  -c, --config=<value>    Path of the external config
-  -d, --data-dir=<value>  Path where the data is stored
-  --columns=<value>       only show provided columns (comma-separated)
   --copy-dir              Create backup from the original data.
   --copy-path=<value>     Provide the path to backup the copied data
-  --csv                   output is csv format [alias: --output=csv]
-  --filter=<value>        filter property by partial string matching, ex: name=foo
+  --fix-only=<option>...  Provide the list of fix options
+                          <options: reference|global_field|json:rte|json:custom-field|blocks|group>
   --modules=<option>...   Provide the list of modules to be audited
                           <options: content-types|global-fields|entries>
-  --no-truncate           do not truncate output to fit screen
   --report-path=<value>   Path to store the audit reports
-  --sort=<value>          property to sort by (prepend '-' for descending)
+
+COMMON FLAGS
+  -c, --config=<value>    Path of the external config
+  -d, --data-dir=<value>  Path where the data is stored
+
+TABLE FLAGS
+  --columns=<value>  Show only the specified columns (comma-separated)
+  --csv              The output is in the CSV format [alias: --output=csv]
+  --filter=<value>   Filter property by partial string matching. For example: name=foo
+  --no-truncate      The output is not truncated to fit the screen
+  --sort=<value>     Property to sort by (prepend '-' for descending)
 
 DESCRIPTION
   Perform audits and fix possible errors in the exported Contentstack data.
@@ -210,6 +234,8 @@ EXAMPLES
 
   $ csdx cm:stacks:audit:fix --report-path=<path> --copy-dir --csv
 
+  $ csdx cm:stacks:audit:fix --fix-only=reference,global_field --copy-dir
+
   $ csdx cm:stacks:audit:fix --report-path=<path> --filter="name=<filter-value>"
 
   $ csdx cm:stacks:audit:fix --report-path=<path> --modules=content-types --filter="name="<filter-value>" --copy-dir --copy-path=<path>

diff --git a/packages/contentstack-audit/package.json b/packages/contentstack-audit/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@contentstack/cli-audit",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "description": "Contentstack audit plugin",
   "author": "Contentstack CLI",
   "homepage": "https://github.com/contentstack/cli",
@@ -18,8 +18,8 @@
     "/oclif.manifest.json"
   ],
   "dependencies": {
-    "@contentstack/cli-command": "~1.2.15",
-    "@contentstack/cli-utilities": "~1.5.5",
+    "@contentstack/cli-command": "~1.2.16",
+    "@contentstack/cli-utilities": "~1.5.7",
     "@oclif/plugin-help": "^5",
     "@oclif/plugin-plugins": "^3.8.4",
     "chalk": "^4.1.2",
@@ -30,6 +30,7 @@
     "winston": "^3.10.0"
   },
   "devDependencies": {
+    "@contentstack/cli-dev-dependencies": "^1.2.4",
     "@oclif/test": "^2.0.3",
     "@types/chai": "^4.3.5",
     "@types/fs-extra": "^11.0.2",
@@ -41,8 +42,10 @@
     "eslint-config-oclif-typescript": "^1.0.3",
     "mocha": "^10.2.0",
     "nyc": "^15.1.0",
-    "oclif": "^3.10.0",
+    "oclif": "^3",
     "shx": "^0.3.4",
+    "sinon": "^17.0.0",
+    "ts-jest": "^29.1.1",
     "ts-node": "^10.9.1",
     "tslib": "^2.5.3",
     "typescript": "^5.1.3"

diff --git a/packages/contentstack-audit/src/audit-base-command.ts b/packages/contentstack-audit/src/audit-base-command.ts
@@ -6,17 +6,17 @@ import isEmpty from 'lodash/isEmpty';
 import { join, resolve } from 'path';
 import cloneDeep from 'lodash/cloneDeep';
 import { cliux, ux } from '@contentstack/cli-utilities';
-import { createWriteStream, existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { createWriteStream, existsSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'fs';
 
 import config from './config';
 import { print } from './util/log';
 import { auditMsg } from './messages';
 import { BaseCommand } from './base-command';
 import { Entries, GlobalField, ContentType } from './modules';
-import { ContentTypeStruct, OutputColumn, RefErrorReturnType } from './types';
+import { CommandNames, ContentTypeStruct, OutputColumn, RefErrorReturnType } from './types';
 
 export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseCommand> {
-  private currentCommand!: string;
+  private currentCommand!: CommandNames;
 
   get fixStatus() {
     return {
@@ -36,7 +36,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
    * @param {string} command - The `command` parameter is a string that represents the current command
    * being executed.
    */
-  async start(command: string): Promise<void> {
+  async start(command: CommandNames): Promise<void> {
     this.currentCommand = command;
     await this.promptQueue();
     await this.createBackUp();
@@ -59,6 +59,11 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
     } else {
       this.log(this.messages.NO_MISSING_REF_FOUND, 'info');
       this.log('');
+
+      if (this.currentCommand === 'cm:stacks:audit:fix' && existsSync(this.sharedConfig.basePath)) {
+        // NOTE Clean up the backup dir if no issue found while audit the content
+        rmSync(this.sharedConfig.basePath, { recursive: true });
+      }
     }
   }
 
@@ -277,15 +282,11 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
    * @returns The function `prepareCSV` returns a Promise that resolves to `void`.
    */
   prepareCSV(moduleName: keyof typeof config.moduleConfig, listOfMissingRefs: Record<string, any>): Promise<void> {
-    const csvStream = csv.format({ headers: true });
     const csvPath = join(this.sharedConfig.reportPath, `${moduleName}.csv`);
-    const assetFileStream = createWriteStream(csvPath);
-    assetFileStream.on('error', (error) => {
-      throw error;
-    });
 
     return new Promise<void>((resolve, reject) => {
-      csvStream.pipe(assetFileStream).on('close', resolve).on('error', reject);
+      // file deepcode ignore MissingClose: Will auto close once csv stream end
+      const ws = createWriteStream(csvPath).on('error', reject);
       const defaultColumns = Object.keys(OutputColumn);
       const userDefinedColumns = this.sharedConfig.flags.columns ? this.sharedConfig.flags.columns.split(',') : null;
       let missingRefs: RefErrorReturnType[] = Object.values(listOfMissingRefs).flat();
@@ -298,6 +299,8 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
         missingRefs = missingRefs.filter((row: RefErrorReturnType) => row[OutputColumn[column]] === value);
       }
 
+      const rowData: Record<string, string | string[]>[] = [];
+
       for (const issue of missingRefs) {
         let row: Record<string, string | string[]> = {};
 
@@ -310,10 +313,10 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
           row['Fix status'] = row.fixStatus;
         }
 
-        csvStream.write(row);
+        rowData.push(row);
       }
 
-      csvStream.end();
+      csv.write(rowData, { headers: true }).pipe(ws).on('error', reject).on('finish', resolve);
     });
   }
 }
diff --git a/packages/contentstack-audit/src/base-command.ts b/packages/contentstack-audit/src/base-command.ts
@@ -1,3 +1,4 @@
+import merge from 'lodash/merge';
 import { existsSync, readFileSync } from 'fs';
 import { Command } from '@contentstack/cli-command';
 import { Flags, FlagInput, Interfaces, cliux as ux } from '@contentstack/cli-utilities';
@@ -27,10 +28,12 @@ export abstract class BaseCommand<T extends typeof Command> extends Command {
   static baseFlags: FlagInput = {
     config: Flags.string({
       char: 'c',
+      helpGroup: 'COMMON',
       description: commonMsg.CONFIG,
     }),
     'data-dir': Flags.string({
       char: 'd',
+      helpGroup: 'COMMON',
       description: commonMsg.DATA_DIR,
     }),
   };
@@ -92,7 +95,10 @@ export abstract class BaseCommand<T extends typeof Command> extends Command {
   registerConfig() {
     if (this.flags.config && existsSync(this.flags.config)) {
       try {
-        this.sharedConfig = JSON.parse(readFileSync(this.flags.config, { encoding: 'utf-8' }));
+        this.sharedConfig = merge(
+          this.sharedConfig,
+          JSON.parse(readFileSync(this.flags.config, { encoding: 'utf-8' })),
+        );
       } catch (error) {
         this.log(error, 'error');
       }

diff --git a/packages/contentstack-audit/src/commands/cm/stacks/audit/fix.ts b/packages/contentstack-audit/src/commands/cm/stacks/audit/fix.ts
@@ -1,6 +1,7 @@
 import { FlagInput, Flags, ux } from '@contentstack/cli-utilities';
 
 import config from '../../../../config';
+import { getTableFlags } from '../../../../util';
 import { auditFixMsg, auditMsg } from '../../../../messages';
 import { AuditBaseCommand } from '../../../../audit-base-command';
 
@@ -13,6 +14,7 @@ export default class AuditFix extends AuditBaseCommand {
     '$ <%= config.bin %> <%= command.id %> --copy-dir',
     '$ <%= config.bin %> <%= command.id %> --report-path=<path> --copy-dir',
     '$ <%= config.bin %> <%= command.id %> --report-path=<path> --copy-dir --csv',
+    '$ <%= config.bin %> <%= command.id %> --fix-only=reference,global_field --copy-dir',
     '$ <%= config.bin %> <%= command.id %> --report-path=<path> --filter="name=<filter-value>"',
     '$ <%= config.bin %> <%= command.id %> --report-path=<path> --modules=content-types --filter="name="<filter-value>" --copy-dir --copy-path=<path>',
   ];
@@ -37,14 +39,17 @@ export default class AuditFix extends AuditBaseCommand {
       dependsOn: ['copy-dir'],
       description: auditFixMsg.BKP_PATH,
     }),
+    'fix-only': Flags.string({
+      multiple: true,
+      options: config['fix-fields'],
+      description: auditFixMsg.FIX_OPTIONS,
+    }),
     yes: Flags.boolean({
       char: 'y',
       hidden: true,
       description: 'Use this flag to skip confirmation',
     }),
-    ...ux.table.flags({
-      only: ['columns', 'sort', 'filter', 'csv', 'no-truncate'],
-    }),
+    ...getTableFlags(),
   };
 
   /**

diff --git a/packages/contentstack-audit/src/commands/cm/stacks/audit/index.ts b/packages/contentstack-audit/src/commands/cm/stacks/audit/index.ts
@@ -3,6 +3,7 @@ import { FlagInput, Flags, ux } from '@contentstack/cli-utilities';
 import config from '../../../../config';
 import { auditMsg } from '../../../../messages';
 import { AuditBaseCommand } from '../../../../audit-base-command';
+import { getTableFlags } from '../../../../util';
 
 export default class Audit extends AuditBaseCommand {
   static aliases: string[] = ['audit', 'cm:stacks:audit'];
@@ -30,9 +31,7 @@ export default class Audit extends AuditBaseCommand {
       options: config.modules,
       description: auditMsg.MODULES,
     }),
-    ...ux.table.flags({
-      only: ['columns', 'sort', 'filter', 'csv', 'no-truncate'],
-    }),
+    ...getTableFlags(),
   };
 
   /**
@@ -44,7 +43,6 @@ export default class Audit extends AuditBaseCommand {
       await this.start('cm:stacks:audit');
     } catch (error) {
       this.log(error instanceof Error ? error.message : error, 'error');
-      console.trace(error);
       ux.action.stop('Process failed.!');
       this.exit(1);
     }

diff --git a/packages/contentstack-audit/src/config/index.ts b/packages/contentstack-audit/src/config/index.ts
@@ -2,6 +2,7 @@ const config = {
   showTerminalOutput: true,
   skipRefs: ['sys_assets'],
   modules: ['content-types', 'global-fields', 'entries'],
+  'fix-fields': ['reference', 'global_field', 'json:rte', 'json:custom-field', 'blocks', 'group'],
   moduleConfig: {
     'content-types': {
       name: 'content type',