Merge pull request #1063 from contentstack/feat/CS-41397

Feat: Functionality to detect missing references for entries

packages/contentstack-audit/README.md

@@ -20,7 +20,7 @@ $ npm install -g @contentstack/cli-audit
 $ csdx COMMAND
 running command...
 $ csdx (--version|-v)
-@contentstack/cli-audit/0.0.0-alpha darwin-arm64 node-v16.19.0
+@contentstack/cli-audit/0.0.0-alpha darwin-arm64 node-v20.7.0
 $ csdx --help [COMMAND]
 USAGE
   $ csdx COMMAND
@@ -74,8 +74,9 @@ Audit and find possible errors in the exported data
 
 ```
 USAGE
-  $ csdx cm:stacks:audit [-c <value>] [-d <value>] [--report-path <value>] [--reference-only] [--modules
-    content-types|global-fields] [--columns <value> | ] [--sort <value>] [--filter <value>] [--csv | --no-truncate]
+  $ csdx cm:stacks:audit [-c <value>] [-d <value>] [--report-path <value>] [--modules
+    content-types|global-fields|entries] [--columns <value> | ] [--sort <value>] [--filter <value>] [--csv |
+    --no-truncate]
 
 FLAGS
   -c, --config=<value>    Path of the external config
@@ -84,9 +85,8 @@ FLAGS
   --csv                   output is csv format [alias: --output=csv]
   --filter=<value>        filter property by partial string matching, ex: name=foo
   --modules=<option>...   Provide list of modules to be audited
-                          <options: content-types|global-fields>
+                          <options: content-types|global-fields|entries>
   --no-truncate           do not truncate output to fit screen
-  --reference-only        Checks only for missing references
   --report-path=<value>   Path to store the audit reports
   --sort=<value>          property to sort by (prepend '-' for descending)
 

packages/contentstack-audit/src/commands/cm/stacks/audit/index.ts

@@ -1,14 +1,15 @@
 import chalk from 'chalk';
 import * as csv from 'fast-csv';
+import isEmpty from 'lodash/isEmpty';
 import { join, resolve } from 'path';
 import { FlagInput, Flags, cliux, ux } from '@contentstack/cli-utilities';
 import { createWriteStream, existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
 
 import config from '../../../../config';
+import { print } from '../../../../util/log';
 import { auditMsg } from '../../../../messages';
 import { BaseCommand } from '../../../../base-command';
-import ContentType from '../../../../modules/content-types';
-import GlobalField from '../../../../modules/global-fields';
+import { Entries, GlobalField, ContentType } from '../../../../modules';
 import { ContentTypeStruct, OutputColumn, RefErrorReturnType } from '../../../../types';
 
 export default class Audit extends BaseCommand<typeof Audit> {
@@ -29,6 +30,7 @@ export default class Audit extends BaseCommand<typeof Audit> {
       description: auditMsg.REPORT_PATH,
     }),
     'reference-only': Flags.boolean({
+      hidden: true,
       description: auditMsg.REFERENCE_ONLY,
     }),
     modules: Flags.string({
@@ -51,13 +53,14 @@ export default class Audit extends BaseCommand<typeof Audit> {
       const reportPath = this.flags['report-path'] || process.cwd();
       this.sharedConfig.reportPath = resolve(reportPath, 'audit-report');
       let { ctSchema, gfSchema } = this.getCtAndGfSchema();
+      let missingCtRefs, missingGfRefs, missingEntryRefs;
 
       for (const module of this.sharedConfig.flags.modules || this.sharedConfig.modules) {
         ux.action.start(this.$t(this.messages.AUDIT_START_SPINNER, { module }));
 
         switch (module) {
           case 'content-types':
-            const missingCtRefs = await new ContentType({
+            missingCtRefs = await new ContentType({
               ctSchema,
               gfSchema,
               log: this.log,
@@ -66,10 +69,9 @@ export default class Audit extends BaseCommand<typeof Audit> {
             }).run();
 
             await this.prepareReport(module, missingCtRefs);
-            this.showOutputOnScreen(missingCtRefs);
             break;
           case 'global-fields':
-            const missingGfRefs = await new GlobalField({
+            missingGfRefs = await new GlobalField({
               ctSchema,
               gfSchema,
               log: this.log,
@@ -78,19 +80,37 @@ export default class Audit extends BaseCommand<typeof Audit> {
             }).run();
 
             await this.prepareReport(module, missingGfRefs);
-            this.showOutputOnScreen(missingGfRefs);
             break;
           case 'entries':
+            missingEntryRefs = await new Entries({
+              ctSchema,
+              gfSchema,
+              log: this.log,
+              moduleName: module,
+              config: this.sharedConfig,
+            }).run();
+            await this.prepareReport(module, missingEntryRefs);
             break;
         }
 
         ux.action.stop();
-        this.log('');
       }
 
-      this.log(this.$t(auditMsg.FINAL_REPORT_PATH, { path: this.sharedConfig.reportPath }), 'warn');
+      this.showOutputOnScreen([
+        { module: 'Content types', missingRefs: missingCtRefs },
+        { module: 'Global Fields', missingRefs: missingGfRefs },
+        { module: 'Entries', missingRefs: missingEntryRefs },
+      ]);
+
+      if (!isEmpty(missingCtRefs) || !isEmpty(missingGfRefs) || !isEmpty(missingEntryRefs)) {
+        this.log(this.$t(auditMsg.FINAL_REPORT_PATH, { path: this.sharedConfig.reportPath }), 'warn');
+      } else {
+        this.log(this.messages.NO_MISSING_REF_FOUND, 'info');
+        this.log('');
+      }
     } catch (error) {
       this.log(error instanceof Error ? error.message : error, 'error');
+      console.trace(error);
       ux.action.stop('Process failed.!');
       this.exit(1);
     }
@@ -143,83 +163,106 @@ export default class Audit extends BaseCommand<typeof Audit> {
   }
 
   /**
-   * The function `showOutputOnScreen` displays data in a table format on the screen, with specific
-   * column headers and formatting options.
-   * @param data - The `data` parameter is a record object that contains information about fields. Each
-   * field is represented by a key-value pair, where the key is the field name and the value is an
-   * object containing various properties such as `name`, `display_name`, `data_type`, `missingRefs`,
-   * and `
+   * The function `showOutputOnScreen` displays missing references on the terminal screen if the
+   * `showTerminalOutput` flag is set to true.
+   * @param {{ module: string; missingRefs?: Record<string, any> }[]} allMissingRefs - An array of
+   * objects, where each object has two properties:
    */
-  showOutputOnScreen(data: Record<string, any>) {
-    ux.table(
-      Object.values(data).flat(),
-      {
-        name: {
-          minWidth: 7,
-          header: 'Title',
-        },
-        display_name: {
-          minWidth: 7,
-          header: 'Field name',
-        },
-        data_type: {
-          minWidth: 7,
-          header: 'Field type',
-        },
-        missingRefs: {
-          minWidth: 7,
-          header: 'Missing references',
-          get: (row) => {
-            return chalk.red(row.missingRefs);
-          },
-        },
-        treeStr: {
-          minWidth: 7,
-          header: 'Path',
-        },
-      },
-      {
-        ...this.flags,
-      },
-    );
-    this.log(''); // NOTE add new line in terminal
+  showOutputOnScreen(allMissingRefs: { module: string; missingRefs?: Record<string, any> }[]) {
+    if (this.sharedConfig.showTerminalOutput) {
+      this.log(''); // NOTE adding new line
+      for (const { module, missingRefs } of allMissingRefs) {
+        if (!isEmpty(missingRefs)) {
+          print([
+            {
+              bold: true,
+              color: 'cyan',
+              message: ` ${module}`,
+            },
+          ]);
+          ux.table(
+            Object.values(missingRefs).flat(),
+            {
+              name: {
+                minWidth: 7,
+                header: 'Title',
+              },
+              display_name: {
+                minWidth: 7,
+                header: 'Field name',
+              },
+              data_type: {
+                minWidth: 7,
+                header: 'Field type',
+              },
+              missingRefs: {
+                minWidth: 7,
+                header: 'Missing references',
+                get: (row) => {
+                  return chalk.red(
+                    typeof row.missingRefs === 'object' ? JSON.stringify(row.missingRefs) : row.missingRefs,
+                  );
+                },
+              },
+              treeStr: {
+                minWidth: 7,
+                header: 'Path',
+              },
+            },
+            {
+              ...this.flags,
+            },
+          );
+          this.log(''); // NOTE adding new line
+        }
+      }
+    }
   }
 
   /**
-   * The `prepareReport` function takes a module name and a list of missing references, and generates a
-   * report in both JSON and CSV formats.
+   * The function prepares a report by writing a JSON file and a CSV file with a list of missing
+   * references for a given module.
    * @param moduleName - The `moduleName` parameter is a string that represents the name of a module.
-   * It is used to generate the filenames for the report files (JSON and CSV).
+   * It is used to generate the filename for the report.
    * @param listOfMissingRefs - The `listOfMissingRefs` parameter is a record object that contains
-   * information about missing references. Each key in the record represents a reference, and the
-   * corresponding value is an array of objects that contain details about the missing reference.
+   * information about missing references. It is a key-value pair where the key represents the
+   * reference name and the value represents additional information about the missing reference.
    * @returns The function `prepareReport` returns a Promise that resolves to `void`.
    */
   prepareReport(moduleName: keyof typeof config.moduleConfig, listOfMissingRefs: Record<string, any>): Promise<void> {
-    return new Promise<void>((resolve, reject) => {
-      if (!existsSync(this.sharedConfig.reportPath)) {
-        mkdirSync(this.sharedConfig.reportPath, { recursive: true });
-      }
+    if (isEmpty(listOfMissingRefs)) return Promise.resolve(void 0);
+
+    if (!existsSync(this.sharedConfig.reportPath)) {
+      mkdirSync(this.sharedConfig.reportPath, { recursive: true });
+    }
 
-      // NOTE write int json
-      writeFileSync(join(this.sharedConfig.reportPath, `${moduleName}.json`), JSON.stringify(listOfMissingRefs));
-      this.log(''); // NOTE add new line in terminal
-
-      // NOTE write into CSV
-      const csvStream = csv.format({ headers: true });
-      const csvPath = join(this.sharedConfig.reportPath, `${moduleName}.csv`);
-      const assetFileStream = createWriteStream(csvPath);
-      assetFileStream.on('error', (error) => {
-        throw error;
-      });
-      csvStream
-        .pipe(assetFileStream)
-        .on('close', () => {
-          resolve();
-          this.log(''); // NOTE add new line in terminal
-        })
-        .on('error', reject);
+    // NOTE write int json
+    writeFileSync(join(this.sharedConfig.reportPath, `${moduleName}.json`), JSON.stringify(listOfMissingRefs));
 
+    // NOTE write into CSV
+    return this.prepareCSV(moduleName, listOfMissingRefs);
+  }
+
+  /**
+   * The function `prepareCSV` takes a module name and a list of missing references, and generates a
+   * CSV file with the specified columns and filtered rows.
+   * @param moduleName - The `moduleName` parameter is a string that represents the name of a module.
+   * It is used to generate the name of the CSV file that will be created.
+   * @param listOfMissingRefs - The `listOfMissingRefs` parameter is a record object that contains
+   * information about missing references. Each key in the record represents a reference, and the
+   * corresponding value is an array of objects that contain details about the missing reference.
+   * @returns The function `prepareCSV` returns a Promise that resolves to `void`.
+   */
+  prepareCSV(moduleName: keyof typeof config.moduleConfig, listOfMissingRefs: Record<string, any>): Promise<void> {
+    const csvStream = csv.format({ headers: true });
+    const csvPath = join(this.sharedConfig.reportPath, `${moduleName}.csv`);
+    const assetFileStream = createWriteStream(csvPath);
+    assetFileStream.on('error', (error) => {
+      throw error;
+    });
+
+    return new Promise<void>((resolve, reject) => {
+      csvStream.pipe(assetFileStream).on('close', resolve).on('error', reject);
       const defaultColumns = Object.keys(OutputColumn);
       const userDefinedColumns = this.sharedConfig.flags.columns ? this.sharedConfig.flags.columns.split(',') : null;
       let missingRefs: RefErrorReturnType[] = Object.values(listOfMissingRefs).flat();
@@ -237,6 +280,7 @@ export default class Audit extends BaseCommand<typeof Audit> {
 
         for (const column of columns) {
           row[column] = issue[OutputColumn[column]];
+          row[column] = typeof row[column] === 'object' ? JSON.stringify(row[column]) : row[column];
         }
 
         csvStream.write(row);

packages/contentstack-audit/src/config/index.ts

@@ -1,6 +1,7 @@
 const config = {
-  modules: ['content-types', 'global-fields'],
+  showTerminalOutput: true,
   skipRefs: ['sys_assets'],
+  modules: ['content-types', 'global-fields', 'entries'],
   moduleConfig: {
     'content-types': {
       name: 'content type',
@@ -12,6 +13,16 @@ const config = {
       dirName: 'global_fields',
       fileName: 'globalfields.json',
     },
+    entries: {
+      name: 'entries',
+      dirName: 'entries',
+      fileName: 'entries.json',
+    },
+    locales: {
+      name: 'locales',
+      dirName: 'locales',
+      fileName: 'locales.json',
+    },
   },
 };
 

packages/contentstack-audit/src/messages/index.ts

@@ -12,10 +12,13 @@ const auditMsg = {
   REPORT_PATH: 'Path to store the audit reports',
   MODULES: 'Provide list of modules to be audited',
   AUDIT_START_SPINNER: 'Starting {module} scanning',
+  PREPARING_ENTRY_METADATA: 'Creating entry metadata',
   REFERENCE_ONLY: 'Checks only for missing references',
   NOT_VALID_PATH: "Provided path: '{path}' is not valid",
-  SCAN_CT_SUCCESS_MSG: "The scanning of {module} '{title}' has been successfully finished.",
+  NO_MISSING_REF_FOUND: 'We have not identified any missing references.',
   FINAL_REPORT_PATH: "Writing reports completed. You can find reports at '{path}'",
+  SCAN_CT_SUCCESS_MSG: "The scanning of {module} '{title}' has been successfully finished.",
+  SCAN_ENTRY_SUCCESS_MSG: "The scanning of {module}({local}) '{title}' has been successfully finished.",
 };
 
 const messages: typeof errors & typeof commonMsg & typeof auditMsg = {

packages/contentstack-audit/src/modules/content-types.ts

@@ -1,4 +1,4 @@
-import { join } from 'path';
+import { resolve } from 'path';
 import find from 'lodash/find';
 import { existsSync } from 'fs';
 
@@ -40,7 +40,7 @@ export default class ContentType {
     this.ctSchema = ctSchema;
     this.gfSchema = gfSchema;
     this.fileName = config.moduleConfig[this.moduleName].fileName;
-    this.folderPath = join(config.basePath, config.moduleConfig[this.moduleName].dirName);
+    this.folderPath = resolve(config.basePath, config.moduleConfig[this.moduleName].dirName);
 
     if (moduleName) this.moduleName = moduleName;
   }
@@ -69,6 +69,12 @@ export default class ContentType {
       );
     }
 
+    for (let propName in this.missingRefs) {
+      if (!this.missingRefs[propName].length) {
+        delete this.missingRefs[propName];
+      }
+    }
+
     return this.missingRefs;
   }