From a5601002aa43c5df37cd3c8d5c994613f6ca3a68 Mon Sep 17 00:00:00 2001 From: Antony Date: Thu, 12 Oct 2023 16:11:17 +0530 Subject: [PATCH 1/3] Feat: Audit fix functionality added for entries module --- packages/contentstack-audit/README.md | 44 +- packages/contentstack-audit/package.json | 2 +- .../src/audit-base-command.ts | 12 +- .../src/commands/cm/stacks/audit/fix.ts | 2 +- .../src/commands/cm/stacks/audit/index.ts | 2 +- .../contentstack-audit/src/messages/index.ts | 3 + .../src/modules/content-types.ts | 325 ++++++++--- .../contentstack-audit/src/modules/entries.ts | 551 +++++++++++++++--- .../src/types/content-types.ts | 19 +- .../contentstack-audit/src/types/entries.ts | 12 +- packages/contentstack/README.md | 14 +- 11 files changed, 793 insertions(+), 193 deletions(-) diff --git a/packages/contentstack-audit/README.md b/packages/contentstack-audit/README.md index 12f951399f..7854aee874 100644 --- a/packages/contentstack-audit/README.md +++ b/packages/contentstack-audit/README.md @@ -12,8 +12,6 @@ $ csdx plugins:install @contentstack/cli-audit ## How to use this plugin -This plugin requires you to be authenticated using [csdx auth:login](https://www.contentstack.com/docs/developers/cli/authenticate-with-the-cli/). - ```sh-session $ npm install -g @contentstack/cli-audit @@ -30,20 +28,24 @@ USAGE # Commands -* [`csdx audit`](#csdx-audit) -* [`csdx audit:fix`](#csdx-auditfix) -* [`csdx cm:stacks:audit`](#csdx-cmstacksaudit) -* [`csdx cm:stacks:audit:fix`](#csdx-cmstacksauditfix) -* [`csdx help [COMMANDS]`](#csdx-help-commands) -* [`csdx plugins`](#csdx-plugins) -* [`csdx plugins:install PLUGIN...`](#csdx-pluginsinstall-plugin) -* [`csdx plugins:inspect PLUGIN...`](#csdx-pluginsinspect-plugin) -* [`csdx plugins:install PLUGIN...`](#csdx-pluginsinstall-plugin-1) -* [`csdx plugins:link PLUGIN`](#csdx-pluginslink-plugin) -* [`csdx plugins:uninstall PLUGIN...`](#csdx-pluginsuninstall-plugin) -* [`csdx plugins:uninstall PLUGIN...`](#csdx-pluginsuninstall-plugin-1) -* [`csdx plugins:uninstall PLUGIN...`](#csdx-pluginsuninstall-plugin-2) -* [`csdx plugins:update`](#csdx-pluginsupdate) +- [@contentstack/cli-audit](#contentstackcli-audit) + - [How to install this plugin](#how-to-install-this-plugin) + - [How to use this plugin](#how-to-use-this-plugin) +- [Commands](#commands) + - [`csdx audit`](#csdx-audit) + - [`csdx audit:fix`](#csdx-auditfix) + - [`csdx cm:stacks:audit`](#csdx-cmstacksaudit) + - [`csdx cm:stacks:audit:fix`](#csdx-cmstacksauditfix) + - [`csdx help [COMMANDS]`](#csdx-help-commands) + - [`csdx plugins`](#csdx-plugins) + - [`csdx plugins:install PLUGIN...`](#csdx-pluginsinstall-plugin) + - [`csdx plugins:inspect PLUGIN...`](#csdx-pluginsinspect-plugin) + - [`csdx plugins:install PLUGIN...`](#csdx-pluginsinstall-plugin-1) + - [`csdx plugins:link PLUGIN`](#csdx-pluginslink-plugin) + - [`csdx plugins:uninstall PLUGIN...`](#csdx-pluginsuninstall-plugin) + - [`csdx plugins:uninstall PLUGIN...`](#csdx-pluginsuninstall-plugin-1) + - [`csdx plugins:uninstall PLUGIN...`](#csdx-pluginsuninstall-plugin-2) + - [`csdx plugins:update`](#csdx-pluginsupdate) ## `csdx audit` @@ -93,14 +95,13 @@ Audit and fix possible errors in the exported data ``` USAGE $ csdx audit:fix [-c ] [-d ] [--report-path ] [--modules - content-types|global-fields|entries] [--backup-dir --copy-dir] [-y] [--columns | ] [--sort ] + content-types|global-fields|entries] [--backup-dir --copy-dir] [--columns | ] [--sort ] [--filter ] [--csv | --no-truncate] FLAGS -c, --config= Path of the external config. -d, --data-dir= Path where the data is stored. - -y, --yes Use this flag to skip confirmation - --backup-dir= Provided path to backup original data + --backup-dir= Provide the path to backup the copied data. --columns= only show provided columns (comma-separated) --copy-dir Create backup from original data --csv output is csv format [alias: --output=csv] @@ -180,14 +181,13 @@ Audit and fix possible errors in the exported data ``` USAGE $ csdx cm:stacks:audit:fix [-c ] [-d ] [--report-path ] [--modules - content-types|global-fields|entries] [--backup-dir --copy-dir] [-y] [--columns | ] [--sort ] + content-types|global-fields|entries] [--backup-dir --copy-dir] [--columns | ] [--sort ] [--filter ] [--csv | --no-truncate] FLAGS -c, --config= Path of the external config. -d, --data-dir= Path where the data is stored. - -y, --yes Use this flag to skip confirmation - --backup-dir= Provided path to backup original data + --backup-dir= Provide the path to backup the copied data. --columns= only show provided columns (comma-separated) --copy-dir Create backup from original data --csv output is csv format [alias: --output=csv] diff --git a/packages/contentstack-audit/package.json b/packages/contentstack-audit/package.json index 7450b84251..f7d5de36c3 100644 --- a/packages/contentstack-audit/package.json +++ b/packages/contentstack-audit/package.json @@ -63,7 +63,7 @@ ], "topics": { "cm:stacks:audit": { - "description": "Audit and find possible errors in the exported data" + "description": "Perform audits and find possible errors in the exported Contentstack data" }, "cm:stacks:audit:fix": { "description": "Audit and fix possible errors in the exported data" diff --git a/packages/contentstack-audit/src/audit-base-command.ts b/packages/contentstack-audit/src/audit-base-command.ts index 99b0f415df..f7a0522914 100644 --- a/packages/contentstack-audit/src/audit-base-command.ts +++ b/packages/contentstack-audit/src/audit-base-command.ts @@ -4,6 +4,7 @@ import { copy } from 'fs-extra'; import { v4 as uuid } from 'uuid'; import isEmpty from 'lodash/isEmpty'; import { join, resolve } from 'path'; +import cloneDeep from 'lodash/cloneDeep'; import { cliux, ux } from '@contentstack/cli-utilities'; import { createWriteStream, existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; @@ -80,15 +81,15 @@ export abstract class AuditBaseCommand extends BaseCommand <%= command.id %> --copy-dir', diff --git a/packages/contentstack-audit/src/commands/cm/stacks/audit/index.ts b/packages/contentstack-audit/src/commands/cm/stacks/audit/index.ts index 897384e672..065d1e91d1 100644 --- a/packages/contentstack-audit/src/commands/cm/stacks/audit/index.ts +++ b/packages/contentstack-audit/src/commands/cm/stacks/audit/index.ts @@ -7,7 +7,7 @@ import { AuditBaseCommand } from '../../../../audit-base-command'; export default class Audit extends AuditBaseCommand { static aliases: string[] = ['audit', 'cm:stacks:audit']; - static description = 'Perform audits and find possible errors in the exported Contentstack data'; + static description = auditMsg.AUDIT_CMD_DESCRIPTION; static examples = [ '$ <%= config.bin %> <%= command.id %>', diff --git a/packages/contentstack-audit/src/messages/index.ts b/packages/contentstack-audit/src/messages/index.ts index 980c6fd153..6ade9f6012 100644 --- a/packages/contentstack-audit/src/messages/index.ts +++ b/packages/contentstack-audit/src/messages/index.ts @@ -6,6 +6,7 @@ const commonMsg = { CONFIG: 'Path of the external config.', CURRENT_WORKING_DIR: 'Current working directory.', DATA_DIR: 'Path where the data is stored.', + FIX_CONFIRMATION: 'Would you like to overwrite existing file.?', }; const auditMsg = { @@ -19,12 +20,14 @@ const auditMsg = { FINAL_REPORT_PATH: "Reports ready. Please find the reports at '{path}'.", SCAN_CT_SUCCESS_MSG: "Successfully completed the scanning of {module} '{title}'.", SCAN_ENTRY_SUCCESS_MSG: "Successfully completed the scanning of {module} ({local}) '{title}'.", + AUDIT_CMD_DESCRIPTION: 'Perform audits and find possible errors in the exported Contentstack data', }; const auditFixMsg = { COPY_DATA: 'Create backup from original data', BKP_PATH: 'Provide the path to backup the copied data.', FIXED_CONTENT_PATH_MAG: 'You can locate the fixed content at {path}', + AUDIT_FIX_CMD_DESCRIPTION: 'Audit and fix possible errors in the exported data', }; const messages: typeof errors & typeof commonMsg & typeof auditMsg & typeof auditFixMsg = { diff --git a/packages/contentstack-audit/src/modules/content-types.ts b/packages/contentstack-audit/src/modules/content-types.ts index adddc133fa..c3f07ad9a1 100644 --- a/packages/contentstack-audit/src/modules/content-types.ts +++ b/packages/contentstack-audit/src/modules/content-types.ts @@ -1,5 +1,7 @@ -import { join, resolve } from 'path'; import find from 'lodash/find'; +import { ux } from '@oclif/core'; +import isEmpty from 'lodash/isEmpty'; +import { join, resolve } from 'path'; import { existsSync, writeFileSync } from 'fs'; import { @@ -15,16 +17,16 @@ import { ModularBlocksDataType, ModuleConstructorParam, ReferenceFieldDataType, + ContentTypeSchemaType, } from '../types'; import auditConfig from '../config'; -import { $t, auditMsg } from '../messages'; -import { ux } from '@oclif/core'; +import { $t, auditMsg, commonMsg } from '../messages'; /* The `ContentType` class is responsible for scanning content types, looking for references, and generating a report in JSON and CSV formats. */ export default class ContentType { public log: LogFn; - private fix: boolean; + protected fix: boolean; public fileName: string; public config: ConfigType; public folderPath: string; @@ -52,7 +54,7 @@ export default class ContentType { * iterates over the schema and looks for references, and returns a list of missing references. * @returns the `missingRefs` object. */ - async run() { + async run(returnFixSchema = false) { if (!existsSync(this.folderPath)) { throw new Error($t(auditMsg.NOT_VALID_PATH, { path: this.folderPath })); } @@ -71,7 +73,13 @@ export default class ContentType { ); } - await this.writeFixContent(); + if (returnFixSchema) { + return this.schema; + } + + if (this.fix) { + await this.writeFixContent(); + } for (let propName in this.missingRefs) { if (!this.missingRefs[propName].length) { @@ -90,7 +98,7 @@ export default class ContentType { let canWrite = true; if (this.fix && !this.config.flags['copy-dir']) { - canWrite = this.config.flags.yes || (await ux.confirm('Would you like to overwrite existing file.?')); + canWrite = this.config.flags.yes || (await ux.confirm(commonMsg.FIX_CONFIRMATION)); } if (canWrite) { @@ -114,47 +122,51 @@ export default class ContentType { */ async lookForReference( tree: Record[], - { schema }: ContentTypeStruct | GlobalFieldDataType | ModularBlockType | GroupFieldDataType, + field: ContentTypeStruct | GlobalFieldDataType | ModularBlockType | GroupFieldDataType, ): Promise { - for (const field of schema ?? []) { - switch (field.data_type) { + if (this.fix) { + field.schema = this.runFixOnSchema(tree, field.schema as ContentTypeSchemaType[]); + } + + for (let child of field.schema ?? []) { + switch (child.data_type) { case 'reference': this.missingRefs[this.currentUid].push( ...this.validateReferenceField( - [...tree, { uid: field.uid, name: field.display_name }], - field as ReferenceFieldDataType, + [...tree, { uid: field.uid, name: child.display_name }], + child as ReferenceFieldDataType, ), ); break; case 'global_field': await this.validateGlobalField( - [...tree, { uid: field.uid, name: field.display_name }], - field as GlobalFieldDataType, + [...tree, { uid: child.uid, name: child.display_name }], + child as GlobalFieldDataType, ); break; case 'json': - if (field.field_metadata.extension) { + if (child.field_metadata.extension) { // NOTE Custom field type - } else if (field.field_metadata.allow_json_rte) { + } else if (child.field_metadata.allow_json_rte) { // NOTE JSON RTE field type this.missingRefs[this.currentUid].push( ...this.validateJsonRTEFields( - [...tree, { uid: field.uid, name: field.display_name }], - field as ReferenceFieldDataType, + [...tree, { uid: child.uid, name: child.display_name }], + child as ReferenceFieldDataType, ), ); } break; case 'blocks': await this.validateModularBlocksField( - [...tree, { uid: field.uid, name: field.display_name }], - field as ModularBlocksDataType, + [...tree, { uid: child.uid, name: child.display_name }], + child as ModularBlocksDataType, ); break; case 'group': await this.validateGroupField( - [...tree, { uid: field.uid, name: field.display_name }], - field as GroupFieldDataType, + [...tree, { uid: child.uid, name: child.display_name }], + child as GroupFieldDataType, ); break; } @@ -184,36 +196,6 @@ export default class ContentType { */ async validateGlobalField(tree: Record[], field: GlobalFieldDataType): Promise { // NOTE Any GlobalField related logic can be added here - let fixStatus; - const missingRefs = []; - const { reference_to, display_name, data_type } = field; - - if (!find(this.gfSchema, { uid: reference_to })) { - missingRefs.push(reference_to); - - if (this.fix) { - try { - delete field.reference_to; - fixStatus = 'Fixed'; - } catch (_error) { - fixStatus = 'Not Fixed'; - } - } - } - - if (missingRefs.length) { - this.missingRefs[this.currentUid].push({ - tree, - data_type, - fixStatus, - missingRefs, - display_name, - ct_uid: this.currentUid, - name: this.currentTitle, - treeStr: tree.map(({ name }) => name).join(' ➜ '), - }); - } - await this.lookForReference(tree, field); } @@ -243,8 +225,8 @@ export default class ContentType { */ async validateModularBlocksField(tree: Record[], field: ModularBlocksDataType): Promise { const { blocks } = field; + this.fixModularBlocksReferences(tree, blocks); - // NOTE Traverse each and every module and look for reference for (const block of blocks) { const { uid, title } = block; @@ -281,7 +263,8 @@ export default class ContentType { tree: Record[], field: ReferenceFieldDataType | JsonRTEFieldDataType, ): RefErrorReturnType[] { - let fixStatus; + if (this.fix) return []; + const missingRefs: string[] = []; let { reference_to, display_name, data_type } = field; @@ -296,23 +279,11 @@ export default class ContentType { } } - if (this.fix) { - try { - field.reference_to = field.reference_to?.filter((ref) => !missingRefs.includes(ref)); - fixStatus = 'Fixed'; - } catch (_error) { - if (this.fix) { - fixStatus = 'Not Fixed'; - } - } - } - return missingRefs.length ? [ { tree, data_type, - fixStatus, missingRefs, display_name, ct_uid: this.currentUid, @@ -325,4 +296,226 @@ export default class ContentType { ] : []; } + + /** + * The function `runFixOnSchema` takes in a tree and a schema, and performs various fixes on the + * schema based on the data types of the fields. + * @param {Record[]} tree - An array of objects representing the tree structure of + * the schema. + * @param {ContentTypeSchemaType[]} schema - The `schema` parameter is an array of + * `ContentTypeSchemaType` objects. Each object represents a field in a content type schema and + * contains properties such as `data_type`, `field_metadata`, `uid`, `name`, etc. + * @returns an array of ContentTypeSchemaType objects. + */ + runFixOnSchema(tree: Record[], schema: ContentTypeSchemaType[]) { + // NOTE Global field Fix + return schema + .map((field) => { + const { data_type } = field; + switch (data_type) { + case 'global_field': + return this.fixGlobalFieldReferences(tree, field as GlobalFieldDataType); + case 'json': + case 'reference': + if (data_type === 'json') { + if (field.field_metadata.extension) { + // NOTE Custom field type + return field; + } else if (field.field_metadata.allow_json_rte) { + return this.fixMissingReferences(tree, field as JsonRTEFieldDataType); + } + } + + return this.fixMissingReferences(tree, field as ReferenceFieldDataType); + case 'blocks': + (field as ModularBlocksDataType).blocks = this.fixModularBlocksReferences( + [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], + (field as ModularBlocksDataType).blocks, + ); + if (isEmpty((field as ModularBlocksDataType).blocks)) { + return null; + } + return field; + case 'group': + return this.fixGroupField(tree, field as GroupFieldDataType); + default: + return field; + } + }) + .filter((val: any) => { + if (val?.schema && isEmpty(val.schema)) return false; + + return !!val; + }) as ContentTypeSchemaType[]; + // (val.schema ? !isEmpty(val.schema) : true) + } + + /** + * The function fixes global field references in a tree structure by adding missing references and + * returning the field if the reference exists, otherwise returning null. + * @param {Record[]} tree - An array of objects representing a tree structure. + * @param {GlobalFieldDataType} field - The `field` parameter is an object that represents a global + * field. It has the following properties: + * @returns either the `field` object if `reference_to` exists in `this.gfSchema`, or `null` if it + * doesn't. + */ + fixGlobalFieldReferences(tree: Record[], field: GlobalFieldDataType) { + const { reference_to, display_name, data_type } = field; + if (reference_to && data_type === 'global_field') { + tree = [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }]; + const refExist = find(this.gfSchema, { uid: reference_to }); + + if (!refExist) { + this.missingRefs[this.currentUid].push({ + tree, + data_type, + display_name, + fixStatus: 'Fixed', + missingRefs: [reference_to], + ct_uid: this.currentUid, + name: this.currentTitle, + treeStr: tree.map(({ name }) => name).join(' ➜ '), + }); + } + + return refExist ? field : null; + } + + return field; + } + + /** + * The function `fixModularBlocksReferences` takes in an array of tree objects and an array of + * modular blocks, and returns an array of modular blocks with fixed references. + * @param {Record[]} tree - An array of objects representing the tree structure. + * @param {ModularBlockType[]} blocks - An array of objects representing modular blocks. Each object + * has properties such as "reference_to", "schema", "title", and "uid". + * @returns an array of `ModularBlockType` objects. + */ + fixModularBlocksReferences(tree: Record[], blocks: ModularBlockType[]) { + return blocks + .map((block) => { + const { reference_to, schema, title: display_name } = block; + tree = [...tree, { uid: block.uid, name: block.title }]; + const refErrorObj = { + tree, + display_name, + fixStatus: 'Fixed', + missingRefs: [reference_to], + ct_uid: this.currentUid, + name: this.currentTitle, + treeStr: tree.map(({ name }) => name).join(' ➜ '), + }; + + if (!schema) { + this.missingRefs[this.currentUid].push(refErrorObj); + + return false; + } + + // NOTE Global field section + if (reference_to) { + const refExist = find(this.gfSchema, { uid: reference_to }); + + if (!refExist) { + this.missingRefs[this.currentUid].push(refErrorObj); + } + + return refExist; + } + + block.schema = this.runFixOnSchema(tree, block.schema as ContentTypeSchemaType[]); + + if (isEmpty(block.schema)) { + this.missingRefs[this.currentUid].push({ + ...refErrorObj, + missingRefs: 'Empty schema found', + treeStr: tree.map(({ name }) => name).join(' ➜ '), + }); + + return null; + } + + return block; + }) + .filter((val) => val) as ModularBlockType[]; + } + + /** + * The function `fixMissingReferences` checks for missing references in a given tree and field, and + * attempts to fix them by removing the missing references from the field's `reference_to` array. + * @param {Record[]} tree - An array of objects representing a tree structure. Each + * object in the array should have a "name" property. + * @param {ReferenceFieldDataType | JsonRTEFieldDataType} field - The `field` parameter is of type + * `ReferenceFieldDataType` or `JsonRTEFieldDataType`. + * @returns the `field` object. + */ + fixMissingReferences(tree: Record[], field: ReferenceFieldDataType | JsonRTEFieldDataType) { + let fixStatus; + const missingRefs: string[] = []; + const { reference_to, data_type, display_name } = field; + + for (const reference of reference_to ?? []) { + // NOTE Can skip specific references keys (Ex, system defined keys can be skipped) + if (this.config.skipRefs.includes(reference)) continue; + + const refExist = find(this.ctSchema, { uid: reference }); + + if (!refExist) { + missingRefs.push(reference); + } + } + + if (!isEmpty(missingRefs)) { + try { + field.reference_to = field.reference_to.filter((ref) => !missingRefs.includes(ref)); + fixStatus = 'Fixed'; + } catch (error) { + fixStatus = `Not Fixed (${JSON.stringify(error)})`; + } + + this.missingRefs[this.currentUid].push({ + tree, + data_type, + fixStatus, + missingRefs, + display_name, + ct_uid: this.currentUid, + name: this.currentTitle, + treeStr: tree.map(({ name }) => name).join(' ➜ '), + }); + } + + return field; + } + + /** + * The function `fixGroupField` takes in an array of objects and a field, and performs some + * operations on the field's schema property. + * @param {Record[]} tree - An array of objects representing a tree structure. + * @param {GroupFieldDataType} field - The `field` parameter is an object that contains the following + * properties: + * @returns The function `fixGroupField` returns either `null` or the `field` object. + */ + fixGroupField(tree: Record[], field: GroupFieldDataType) { + const { data_type, display_name } = field; + field.schema = this.runFixOnSchema(tree, field.schema as ContentTypeSchemaType[]); + + if (isEmpty(field.schema)) { + this.missingRefs[this.currentUid].push({ + tree, + data_type, + display_name, + fixStatus: 'Fixed', + ct_uid: this.currentUid, + name: this.currentTitle, + missingRefs: 'Empty schema found', + treeStr: tree.map(({ name }) => name).join(' ➜ '), + }); + + return null; + } + + return field; + } } diff --git a/packages/contentstack-audit/src/modules/entries.ts b/packages/contentstack-audit/src/modules/entries.ts index 99c3c000dd..ccb141ff40 100644 --- a/packages/contentstack-audit/src/modules/entries.ts +++ b/packages/contentstack-audit/src/modules/entries.ts @@ -1,21 +1,28 @@ import find from 'lodash/find'; +import { ux } from '@oclif/core'; import values from 'lodash/values'; import isEmpty from 'lodash/isEmpty'; import { join, resolve } from 'path'; -import { existsSync, readFileSync } from 'fs'; import { FsUtility } from '@contentstack/cli-utilities'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import auditConfig from '../config'; +import ContentType from './content-types'; +import { $t, auditMsg, commonMsg } from '../messages'; import { LogFn, Locale, ConfigType, EntryStruct, + EntryFieldType, ModularBlockType, ContentTypeStruct, CtConstructorParam, GroupFieldDataType, GlobalFieldDataType, JsonRTEFieldDataType, + ContentTypeSchemaType, + EntryModularBlockType, ModularBlocksDataType, ModuleConstructorParam, ReferenceFieldDataType, @@ -26,11 +33,10 @@ import { EntryModularBlocksDataType, EntryReferenceFieldDataType, } from '../types'; -import auditConfig from '../config'; -import { $t, auditMsg } from '../messages'; export default class Entries { public log: LogFn; + private fix: boolean; public fileName: string; public locales!: Locale[]; public config: ConfigType; @@ -39,19 +45,20 @@ export default class Entries { public currentTitle!: string; public gfSchema: ContentTypeStruct[]; public ctSchema: ContentTypeStruct[]; + private entries!: Record; protected missingRefs: Record = {}; public entryMetaData: Record[] = []; - public moduleName: keyof typeof auditConfig.moduleConfig = 'content-types'; + public moduleName: keyof typeof auditConfig.moduleConfig = 'entries'; - constructor({ log, config, moduleName, ctSchema, gfSchema }: ModuleConstructorParam & CtConstructorParam) { + constructor({ log, fix, config, moduleName, ctSchema, gfSchema }: ModuleConstructorParam & CtConstructorParam) { this.log = log; this.config = config; + this.fix = fix ?? false; this.ctSchema = ctSchema; this.gfSchema = gfSchema; + this.moduleName = moduleName ?? 'entries'; this.fileName = config.moduleConfig[this.moduleName].fileName; this.folderPath = resolve(config.basePath, config.moduleConfig.entries.dirName); - - if (moduleName) this.moduleName = moduleName; } /** @@ -66,21 +73,40 @@ export default class Entries { await this.prepareEntryMetaData(); + this.ctSchema = (await new ContentType({ + fix: true, + log: () => {}, + config: this.config, + moduleName: 'content-types', + ctSchema: this.ctSchema, + gfSchema: this.gfSchema, + }).run(true)) as ContentTypeStruct[]; + this.gfSchema = (await new ContentType({ + fix: true, + log: () => {}, + config: this.config, + moduleName: 'entries', + ctSchema: this.ctSchema, + gfSchema: this.gfSchema, + }).run(true)) as ContentTypeStruct[]; + for (const { code } of this.locales) { for (const ctSchema of this.ctSchema) { const basePath = join(this.folderPath, ctSchema.uid, code); const fsUtility = new FsUtility({ basePath, indexFileName: 'index.json' }); const indexer = fsUtility.indexFileContent; - for (const _ in indexer) { + for (const fileIndex in indexer) { const entries = (await fsUtility.readChunkFiles.next()) as Record; - for (const entryUid in entries) { - let entry = entries[entryUid]; + this.entries = entries; + + for (const entryUid in this.entries) { + const entry = this.entries[entryUid]; const { uid, title } = entry; this.currentUid = uid; this.currentTitle = title; this.missingRefs[this.currentUid] = []; - await this.lookForReference([{ uid, name: title }], ctSchema, entry); + this.lookForReference([{ uid, name: title }], ctSchema, this.entries[entryUid]); this.log( $t(auditMsg.SCAN_ENTRY_SUCCESS_MSG, { title, @@ -90,6 +116,10 @@ export default class Entries { 'info', ); } + + if (this.fix) { + await this.writeFixContent(`${basePath}/${indexer[fileIndex]}`, this.entries); + } } } } @@ -104,6 +134,22 @@ export default class Entries { return this.missingRefs; } + /** + * The function checks if it can write the fix content to a file and if so, it writes the content as + * JSON to the specified file path. + */ + async writeFixContent(filePath: string, schema: Record) { + let canWrite = true; + + if (this.fix && !this.config.flags['copy-dir']) { + canWrite = this.config.flags.yes || (await ux.confirm(commonMsg.FIX_CONFIRMATION)); + } + + if (canWrite) { + writeFileSync(filePath, JSON.stringify(schema)); + } + } + /** * The function `lookForReference` iterates over a given schema and validates different field types * such as reference, global field, JSON, modular blocks, and group fields. @@ -116,54 +162,61 @@ export default class Entries { * EntryGroupFieldDataType} entry - The `entry` parameter is an object that represents the data of an * entry. It can have different types depending on the `schema` parameter. */ - async lookForReference( + lookForReference( tree: Record[], - { schema }: ContentTypeStruct | GlobalFieldDataType | ModularBlockType | GroupFieldDataType, - entry: EntryStruct | EntryGlobalFieldDataType | EntryModularBlocksDataType | EntryGroupFieldDataType, - ): Promise { - for (const field of schema ?? []) { - const { uid } = field; - switch (field.data_type) { + field: ContentTypeStruct | GlobalFieldDataType | ModularBlockType | GroupFieldDataType, + entry: EntryFieldType, + ) { + if (this.fix) { + entry = this.runFixOnSchema(tree, field.schema as ContentTypeSchemaType[], entry); + } + + for (const child of field.schema ?? []) { + const { uid } = child; + + if (!entry?.[uid]) return; + + switch (child.data_type) { case 'reference': this.missingRefs[this.currentUid].push( ...this.validateReferenceField( - [...tree, { uid: field.uid, name: field.display_name, field: uid }], - field as ReferenceFieldDataType, + [...tree, { uid: child.uid, name: child.display_name, field: uid }], + child as ReferenceFieldDataType, entry[uid] as EntryReferenceFieldDataType[], ), ); break; case 'global_field': - await this.validateGlobalField( - [...tree, { uid: field.uid, name: field.display_name, field: uid }], - field as GlobalFieldDataType, + this.validateGlobalField( + [...tree, { uid: child.uid, name: child.display_name, field: uid }], + child as GlobalFieldDataType, entry[uid] as EntryGlobalFieldDataType, ); break; case 'json': - if (field.field_metadata.extension) { + if (child.field_metadata.extension) { // NOTE Custom field type - } else if (field.field_metadata.allow_json_rte) { + } else if (child.field_metadata.allow_json_rte) { // NOTE JSON RTE field type - await this.validateJsonRTEFields( - [...tree, { uid: field.uid, name: field.display_name, field: uid }], - field as JsonRTEFieldDataType, + this.validateJsonRTEFields( + [...tree, { uid: child.uid, name: child.display_name, field: uid }], + child as JsonRTEFieldDataType, entry[uid] as EntryJsonRTEFieldDataType, ); } break; case 'blocks': - await this.validateModularBlocksField( - [...tree, { uid: field.uid, name: field.display_name, field: uid }], - field as ModularBlocksDataType, + this.validateModularBlocksField( + [...tree, { uid: child.uid, name: child.display_name, field: uid }], + child as ModularBlocksDataType, entry[uid] as EntryModularBlocksDataType[], ); break; case 'group': - await this.validateGroupField( - [...tree, { uid: field.uid, name: field.display_name, field: uid }], - field as GroupFieldDataType, - entry[uid] as EntryGroupFieldDataType, + this.validateGroupField( + [...tree, { uid: field.uid, name: child.display_name, field: uid }], + child as GroupFieldDataType, + entry[uid] as EntryGroupFieldDataType[], ); break; } @@ -203,13 +256,13 @@ export default class Entries { * @param {EntryGlobalFieldDataType} field - The `field` parameter is of type * `EntryGlobalFieldDataType`. It represents a single global field entry. */ - async validateGlobalField( + validateGlobalField( tree: Record[], fieldStructure: GlobalFieldDataType, field: EntryGlobalFieldDataType, - ): Promise { + ) { // NOTE Any GlobalField related logic can be added here - await this.lookForReference(tree, fieldStructure, field); + this.lookForReference(tree, fieldStructure, field); } /** @@ -224,38 +277,23 @@ export default class Entries { * `EntryJsonRTEFieldDataType`, which represents a JSON RTE field in an entry. It contains properties * such as `uid`, `attrs`, and `children`. */ - async validateJsonRTEFields( + validateJsonRTEFields( tree: Record[], fieldStructure: JsonRTEFieldDataType, field: EntryJsonRTEFieldDataType, ) { + // const missingRefIndex = [] // NOTE Other possible reference logic will be added related to JSON RTE (Ex missing assets, extensions etc.,) - for (const child of field?.children ?? []) { - const { uid: childrenUid, attrs, children } = child; - const { 'entry-uid': entryUid, 'content-type-uid': contentTypeUid } = attrs || {}; - - if (entryUid) { - const refExist = find(this.entryMetaData, { uid: entryUid }); + for (const index in field?.children ?? []) { + const child = field.children[index]; + const { children } = child; - if (!refExist) { - tree.push({ field: 'children' }, { field: childrenUid, uid: fieldStructure.uid }); - this.missingRefs[this.currentUid].push({ - tree, - uid: this.currentUid, - name: this.currentTitle, - data_type: fieldStructure.data_type, - display_name: fieldStructure.display_name, - treeStr: tree - .map(({ name }) => name) - .filter((val) => val) - .join(' ➜ '), - missingRefs: [{ uid: entryUid, 'content-type-uid': contentTypeUid }], - }); - } + if (!this.fix) { + this.jsonRefCheck(tree, fieldStructure, child); } if (!isEmpty(children)) { - await this.validateJsonRTEFields(tree, fieldStructure, child); + this.validateJsonRTEFields(tree, fieldStructure, field.children[index]); } } } @@ -273,21 +311,24 @@ export default class Entries { * @param {EntryModularBlocksDataType[]} field - The `field` parameter is an array of objects of type * `EntryModularBlocksDataType`. */ - async validateModularBlocksField( + validateModularBlocksField( tree: Record[], fieldStructure: ModularBlocksDataType, field: EntryModularBlocksDataType[], - ): Promise { - const { blocks } = fieldStructure; + ) { + if (!this.fix) { + for (const index in field) { + this.modularBlockRefCheck(tree, fieldStructure.blocks, field[index], +index); + } + } - // NOTE Traverse each and every module and look for reference - for (let index = 0; index < blocks.length; index++) { - const ctBlock = blocks[index]; - const entryBlock = field[index]; - const { uid } = ctBlock; + for (const block of fieldStructure.blocks) { + const { uid, title } = block; - if (entryBlock?.[uid]) { - await this.lookForReference([...tree, { field: uid }], ctBlock, entryBlock[uid] as EntryModularBlocksDataType); + for (const eBlock of field) { + if (eBlock[uid]) { + this.lookForReference([...tree, { uid, name: title }], block, eBlock[uid] as EntryModularBlocksDataType); + } } } } @@ -301,13 +342,23 @@ export default class Entries { * @param {EntryGroupFieldDataType} field - The `field` parameter is of type * `EntryGroupFieldDataType` and represents a single group field entry. */ - async validateGroupField( + validateGroupField( tree: Record[], fieldStructure: GroupFieldDataType, - field: EntryGroupFieldDataType, - ): Promise { + field: EntryGroupFieldDataType | EntryGroupFieldDataType[], + ) { // NOTE Any Group Field related logic can be added here (Ex data serialization or picking any metadata for report etc.,) - await this.lookForReference(tree, fieldStructure, field); + if (Array.isArray(field)) { + field.forEach((eGroup) => { + this.lookForReference( + [...tree, { uid: fieldStructure.uid, display_name: fieldStructure.display_name }], + fieldStructure, + eGroup, + ); + }); + } else { + this.lookForReference(tree, fieldStructure, field); + } } /** @@ -330,10 +381,13 @@ export default class Entries { fieldStructure: ReferenceFieldDataType, field: EntryReferenceFieldDataType[], ): EntryRefErrorReturnType[] { - const missingRefs = []; - const { data_type, display_name } = fieldStructure; + if (this.fix) return []; + + const missingRefs: Record[] = []; + const { uid: data_type, display_name } = fieldStructure; - for (const reference of field ?? []) { + for (const index in field ?? []) { + const reference = field[index]; const { uid } = reference; // NOTE Can skip specific references keys (Ex, system defined keys can be skipped) // if (this.config.skipRefs.includes(reference)) continue; @@ -341,7 +395,7 @@ export default class Entries { const refExist = find(this.entryMetaData, { uid }); if (!refExist) { - missingRefs.push(reference as Record); + missingRefs.push(reference); } } @@ -363,6 +417,345 @@ export default class Entries { : []; } + /** + * The function `runFixOnSchema` takes in a tree, schema, and entry, and applies fixes to the entry + * based on the schema. + * @param {Record[]} tree - An array of objects representing the tree structure of + * the schema. Each object has the following properties: + * @param {ContentTypeSchemaType[]} schema - The `schema` parameter is an array of objects + * representing the content type schema. Each object in the array contains information about a + * specific field in the schema, such as its unique identifier (`uid`) and data type (`data_type`). + * @param {EntryFieldType} entry - The `entry` parameter is of type `EntryFieldType`, which + * represents the data of an entry. It is an object that contains fields as key-value pairs, where + * the key is the field UID (unique identifier) and the value is the field data. + * @returns the updated `entry` object after applying fixes to the fields based on the provided + * `schema`. + */ + runFixOnSchema(tree: Record[], schema: ContentTypeSchemaType[], entry: EntryFieldType) { + // NOTE Global field Fix + schema.forEach((field) => { + const { uid, data_type } = field; + + switch (data_type) { + case 'global_field': + entry[uid] = this.fixGlobalFieldReferences( + [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], + field as GlobalFieldDataType, + entry[uid] as EntryGlobalFieldDataType, + ) as EntryGlobalFieldDataType; + break; + case 'json': + case 'reference': + if (data_type === 'json') { + if (field.field_metadata.extension) { + // NOTE Custom field type + return field; + } else if (field.field_metadata.allow_json_rte) { + return this.fixJsonRteMissingReferences( + [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], + field as JsonRTEFieldDataType, + entry[uid] as EntryJsonRTEFieldDataType, + ); + } + } + + // NOTE Reference field + entry[uid] = this.fixMissingReferences( + [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], + field as ReferenceFieldDataType, + entry[uid] as EntryReferenceFieldDataType[], + ) as EntryReferenceFieldDataType[]; + break; + case 'blocks': + entry[uid] = this.fixModularBlocksReferences( + [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], + (field as ModularBlocksDataType).blocks, + entry[uid] as EntryModularBlocksDataType[], + ); + break; + case 'group': + entry[uid] = this.fixGroupField( + [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], + field as GroupFieldDataType, + entry[uid] as EntryGroupFieldDataType[], + ) as EntryGroupFieldDataType; + break; + } + }); + + return entry; + } + + /** + * The function `fixGlobalFieldReferences` adds a new entry to a tree data structure and runs a fix + * on the schema. + * @param {Record[]} tree - An array of objects representing the tree structure. + * @param {GlobalFieldDataType} field - The `field` parameter is of type `GlobalFieldDataType` and + * represents a global field object. It contains properties such as `uid` and `display_name`. + * @param {EntryGlobalFieldDataType} entry - The `entry` parameter is of type + * `EntryGlobalFieldDataType` and represents the global field entry that needs to be fixed. + * @returns the result of calling the `runFixOnSchema` method with the updated `tree` array, + * `field.schema`, and `entry` as arguments. + */ + fixGlobalFieldReferences( + tree: Record[], + field: GlobalFieldDataType, + entry: EntryGlobalFieldDataType, + ) { + return this.runFixOnSchema([...tree, { uid: field.uid, display_name: field.display_name }], field.schema, entry); + } + + /** + * The function `fixModularBlocksReferences` takes in a tree, a list of blocks, and an entry, and + * performs various operations to fix references within the entry. + * @param {Record[]} tree - An array of objects representing the tree structure of + * the modular blocks. + * @param {ModularBlockType[]} blocks - An array of objects representing modular blocks. Each object + * has properties like `uid` (unique identifier) and `title` (display name). + * @param {EntryModularBlocksDataType[]} entry - An array of objects representing the modular blocks + * data in an entry. Each object in the array represents a modular block and contains its unique + * identifier (uid) and other properties. + * @returns the updated `entry` array after performing some modifications. + */ + fixModularBlocksReferences( + tree: Record[], + blocks: ModularBlockType[], + entry: EntryModularBlocksDataType[], + ) { + entry = entry + .map((block, index) => this.modularBlockRefCheck(tree, blocks, block, index)) + .filter((val) => !isEmpty(val)); + + blocks.forEach((block) => { + entry = entry + .map((eBlock) => { + if (!isEmpty(block.schema)) { + if (eBlock[block.uid]) { + eBlock[block.uid] = this.runFixOnSchema( + [...tree, { uid: block.uid, display_name: block.title }], + block.schema as ContentTypeSchemaType[], + eBlock[block.uid] as EntryFieldType, + ) as EntryModularBlockType; + } + } + + return eBlock; + }) + .filter((val) => !isEmpty(val)) as EntryModularBlocksDataType[]; + }); + + return entry; + } + + /** + * The function `fixGroupField` takes in a tree, a field, and an entry, and if the field has a + * schema, it runs a fix on the schema and returns the updated entry, otherwise it returns the + * original entry. + * @param {Record[]} tree - An array of objects representing the tree structure. + * @param {GroupFieldDataType} field - The `field` parameter is of type `GroupFieldDataType` and + * represents a group field object. It contains properties such as `uid` (unique identifier) and + * `display_name` (name of the field). + * @param {EntryGroupFieldDataType} entry - The `entry` parameter is of type + * `EntryGroupFieldDataType`. + * @returns If the `field.schema` is not empty, the function will return the result of calling + * `this.runFixOnSchema` with the updated `tree`, `field.schema`, and `entry` as arguments. + * Otherwise, it will return the `entry` as is. + */ + fixGroupField( + tree: Record[], + field: GroupFieldDataType, + entry: EntryGroupFieldDataType | EntryGroupFieldDataType[], + ) { + if (!isEmpty(field.schema)) { + if (Array.isArray(entry)) { + entry = entry.map((eGroup) => { + return this.runFixOnSchema( + [...tree, { uid: field.uid, display_name: field.display_name }], + field.schema as ContentTypeSchemaType[], + eGroup, + ); + }) as EntryGroupFieldDataType[]; + } else { + entry = this.runFixOnSchema( + [...tree, { uid: field.uid, display_name: field.display_name }], + field.schema as ContentTypeSchemaType[], + entry, + ) as EntryGroupFieldDataType; + } + } + + return entry; + } + + /** + * The function fixes missing references in a JSON tree structure. + * @param {Record[]} tree - An array of objects representing a tree structure. Each + * object in the array has a string key and an unknown value. + * @param {ReferenceFieldDataType | JsonRTEFieldDataType} field - The `field` parameter can be of + * type `ReferenceFieldDataType` or `JsonRTEFieldDataType`. + * @param {EntryJsonRTEFieldDataType} entry - The `entry` parameter is of type + * `EntryJsonRTEFieldDataType`, which represents an entry in a JSON Rich Text Editor (JsonRTE) field. + * @returns the updated `entry` object with fixed missing references in the `children` property. + */ + fixJsonRteMissingReferences( + tree: Record[], + field: ReferenceFieldDataType | JsonRTEFieldDataType, + entry: EntryJsonRTEFieldDataType, + ) { + entry.children = entry.children + .map((child) => { + const refExist = this.jsonRefCheck(tree, field, child); + + if (!refExist) return null; + + if (isEmpty(child.children)) { + child = this.fixJsonRteMissingReferences(tree, field, child); + } + + return child; + }) + .filter((val) => val) as EntryJsonRTEFieldDataType[]; + + return entry; + } + + /** + * The `fixMissingReferences` function checks for missing references in an entry and adds them to a + * list if they are not found. + * @param {Record[]} tree - An array of objects representing a tree structure. Each + * object in the array should have a "name" property and an optional "index" property. + * @param {ReferenceFieldDataType | JsonRTEFieldDataType} field - The `field` parameter is of type + * `ReferenceFieldDataType` or `JsonRTEFieldDataType`. + * @param {EntryReferenceFieldDataType[]} entry - The `entry` parameter is an array of objects that + * represent references to other entries. Each object in the array has the following properties: + * @returns the `entry` variable. + */ + fixMissingReferences( + tree: Record[], + field: ReferenceFieldDataType | JsonRTEFieldDataType, + entry: EntryReferenceFieldDataType[], + ) { + const missingRefs: Record[] = []; + entry = entry + .map((reference) => { + const { uid } = reference; + const refExist = find(this.entryMetaData, { uid }); + + if (!refExist) { + missingRefs.push(reference); + return null; + } + + return reference; + }) + .filter((val) => val) as EntryReferenceFieldDataType[]; + + if (!isEmpty(missingRefs)) { + this.missingRefs[this.currentUid].push({ + tree, + fixStatus: 'Fixed', + uid: this.currentUid, + name: this.currentTitle, + data_type: field.data_type, + display_name: field.display_name, + treeStr: tree + .map(({ name, index }) => (index || index === 0 ? `[${index}].${name}` : name)) + .filter((val) => val) + .join(' ➜ '), + missingRefs, + }); + } + + return entry; + } + + /** + * The function `modularBlockRefCheck` checks for invalid keys in an entry block and returns the + * updated entry block. + * @param {Record[]} tree - An array of objects representing the tree structure of + * the blocks. + * @param {ModularBlockType[]} blocks - The `blocks` parameter is an array of `ModularBlockType` + * objects. + * @param {EntryModularBlocksDataType} entryBlock - The `entryBlock` parameter is an object that + * represents a modular block entry. It contains key-value pairs where the keys are the UIDs of the + * modular blocks and the values are the data associated with each modular block. + * @param {Number} index - The `index` parameter is a number that represents the index of the current + * block in the `tree` array. + * @returns the `entryBlock` object. + */ + modularBlockRefCheck( + tree: Record[], + blocks: ModularBlockType[], + entryBlock: EntryModularBlocksDataType, + index: Number, + ) { + const validBlockUid = blocks.map((block) => block.uid); + const invalidKeys = Object.keys(entryBlock).filter((key) => !validBlockUid.includes(key)); + + invalidKeys.forEach((key) => { + if (this.fix) { + delete entryBlock[key]; + } + + this.missingRefs[this.currentUid].push({ + uid: this.currentUid, + name: this.currentTitle, + data_type: key, + display_name: key, + fixStatus: this.fix ? 'Fixed' : undefined, + tree: [...tree, { index, uid: key, name: key }], + treeStr: [...tree, { index, uid: key, name: key }] + .map(({ name, index }) => (index || index === 0 ? `[${index}].${name}` : name)) + .filter((val) => val) + .join(' ➜ '), + missingRefs: [key], + }); + }); + + return entryBlock; + } + + /** + * The `jsonRefCheck` function checks if a reference exists in a JSON tree and adds missing + * references to a list if they are not found. + * @param {Record[]} tree - An array of objects representing the tree structure. + * @param {JsonRTEFieldDataType} schema - The `schema` parameter is of type `JsonRTEFieldDataType` + * and represents the schema of a JSON field. It contains properties such as `uid`, `data_type`, and + * `display_name`. + * @param {EntryJsonRTEFieldDataType} child - The `child` parameter is an object that represents a + * child entry in a JSON tree. It has the following properties: + * @returns The function `jsonRefCheck` returns either `null` or `true`. + */ + jsonRefCheck(tree: Record[], schema: JsonRTEFieldDataType, child: EntryJsonRTEFieldDataType) { + const { uid: childrenUid } = child; + const { 'entry-uid': entryUid, 'content-type-uid': contentTypeUid } = child.attrs || {}; + + if (entryUid) { + const refExist = find(this.entryMetaData, { uid: entryUid }); + + if (!refExist) { + tree.push({ field: 'children' }, { field: childrenUid, uid: schema.uid }); + this.missingRefs[this.currentUid].push({ + tree, + uid: this.currentUid, + name: this.currentTitle, + data_type: schema.data_type, + display_name: schema.display_name, + fixStatus: this.fix ? 'Fixed' : undefined, + treeStr: tree + .map(({ name }) => name) + .filter((val) => val) + .join(' ➜ '), + missingRefs: [{ uid: entryUid, 'content-type-uid': contentTypeUid }], + }); + + return null; + } + } + + return true; + } + /** * The function prepares entry metadata by reading and processing files from different locales and * schemas. diff --git a/packages/contentstack-audit/src/types/content-types.ts b/packages/contentstack-audit/src/types/content-types.ts index ef6a6b9e12..187b71188f 100644 --- a/packages/contentstack-audit/src/types/content-types.ts +++ b/packages/contentstack-audit/src/types/content-types.ts @@ -1,18 +1,19 @@ import config from '../config'; import { ConfigType, LogFn } from './utils'; +type ContentTypeSchemaType = + | ReferenceFieldDataType + | GlobalFieldDataType + | CustomFieldDataType + | JsonRTEFieldDataType + | GroupFieldDataType + | ModularBlocksDataType; + type ContentTypeStruct = { uid: string; title: string; description: string; - schema?: ( - | ReferenceFieldDataType - | GlobalFieldDataType - | CustomFieldDataType - | JsonRTEFieldDataType - | GroupFieldDataType - | ModularBlocksDataType - )[]; + schema?: ContentTypeSchemaType[]; }; type ModuleConstructorParam = { @@ -80,6 +81,7 @@ type GroupFieldDataType = CommonDataTypeStruct & { type ModularBlockType = { uid: string; title: string; + reference_to?: string; schema: ( | JsonRTEFieldDataType | ModularBlocksDataType @@ -122,4 +124,5 @@ export { ModularBlocksSchemaTypes, ModularBlockType, OutputColumn, + ContentTypeSchemaType, }; diff --git a/packages/contentstack-audit/src/types/entries.ts b/packages/contentstack-audit/src/types/entries.ts index 0038e9acf7..7e3f7010a5 100644 --- a/packages/contentstack-audit/src/types/entries.ts +++ b/packages/contentstack-audit/src/types/entries.ts @@ -41,8 +41,10 @@ type EntryJsonRTEFieldDataType = { }; // NOTE Type 5 +type GroupFieldType = EntryReferenceFieldDataType[] | EntryGlobalFieldDataType | EntryJsonRTEFieldDataType; + type EntryGroupFieldDataType = { - [key: string]: EntryReferenceFieldDataType[] | EntryGlobalFieldDataType | EntryJsonRTEFieldDataType; + [key: string]: GroupFieldType; }; // NOTE Type 6 @@ -63,14 +65,18 @@ type EntryRefErrorReturnType = { uid: string; treeStr: string; data_type: string; - missingRefs: string[] | Record[]; + fixStatus?: string; display_name: string; tree: Record[]; + missingRefs: string[] | Record[]; }; +type EntryFieldType = EntryStruct | EntryGlobalFieldDataType | EntryModularBlocksDataType | EntryGroupFieldDataType; + export { Locale, EntryStruct, + EntryFieldType, EntryGlobalFieldDataType, EntryCustomFieldDataType, EntryJsonRTEFieldDataType, @@ -78,4 +84,6 @@ export { EntryModularBlocksDataType, EntryReferenceFieldDataType, EntryRefErrorReturnType, + GroupFieldType, + EntryModularBlockType, }; diff --git a/packages/contentstack/README.md b/packages/contentstack/README.md index 93f78d147d..3f42815fca 100644 --- a/packages/contentstack/README.md +++ b/packages/contentstack/README.md @@ -18,7 +18,7 @@ $ npm install -g @contentstack/cli $ csdx COMMAND running command... $ csdx (--version|-v) -@contentstack/cli/1.9.1 darwin-arm64 node-v20.8.0 +@contentstack/cli/1.9.2 darwin-arm64 node-v20.8.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND @@ -157,14 +157,13 @@ Audit and fix possible errors in the exported data ``` USAGE $ csdx audit:fix [-c ] [-d ] [--report-path ] [--modules - content-types|global-fields|entries] [--backup-dir --copy-dir] [-y] [--columns | ] [--sort ] + content-types|global-fields|entries] [--backup-dir --copy-dir] [--columns | ] [--sort ] [--filter ] [--csv | --no-truncate] FLAGS -c, --config= Path of the external config. -d, --data-dir= Path where the data is stored. - -y, --yes Use this flag to skip confirmation - --backup-dir= Provided path to backup original data + --backup-dir= Provide the path to backup the copied data. --columns= only show provided columns (comma-separated) --copy-dir Create backup from original data --csv output is csv format [alias: --output=csv] @@ -2344,14 +2343,13 @@ Audit and fix possible errors in the exported data ``` USAGE $ csdx cm:stacks:audit:fix [-c ] [-d ] [--report-path ] [--modules - content-types|global-fields|entries] [--backup-dir --copy-dir] [-y] [--columns | ] [--sort ] + content-types|global-fields|entries] [--backup-dir --copy-dir] [--columns | ] [--sort ] [--filter ] [--csv | --no-truncate] FLAGS -c, --config= Path of the external config. -d, --data-dir= Path where the data is stored. - -y, --yes Use this flag to skip confirmation - --backup-dir= Provided path to backup original data + --backup-dir= Provide the path to backup the copied data. --columns= only show provided columns (comma-separated) --copy-dir Create backup from original data --csv output is csv format [alias: --output=csv] @@ -2985,7 +2983,7 @@ DESCRIPTION Display help for csdx. ``` -_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v5.2.20/src/commands/help.ts)_ +_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v5.2.14/src/commands/help.ts)_ ## `csdx launch` From 0afa4ca4556a2f9ebe2cfe999cb46d88a3d7ef66 Mon Sep 17 00:00:00 2001 From: Antony Date: Thu, 12 Oct 2023 19:28:33 +0530 Subject: [PATCH 2/3] Fix: Json RTE custom type and empty reference_to files handled --- .../src/modules/content-types.ts | 4 +-- .../contentstack-audit/src/modules/entries.ts | 32 ++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/packages/contentstack-audit/src/modules/content-types.ts b/packages/contentstack-audit/src/modules/content-types.ts index c3f07ad9a1..ed42e2c0c0 100644 --- a/packages/contentstack-audit/src/modules/content-types.ts +++ b/packages/contentstack-audit/src/modules/content-types.ts @@ -344,10 +344,10 @@ export default class ContentType { }) .filter((val: any) => { if (val?.schema && isEmpty(val.schema)) return false; + if (val?.reference_to && isEmpty(val.reference_to)) return false; return !!val; }) as ContentTypeSchemaType[]; - // (val.schema ? !isEmpty(val.schema) : true) } /** @@ -466,7 +466,7 @@ export default class ContentType { } } - if (!isEmpty(missingRefs)) { + if (this.fix && !isEmpty(missingRefs)) { try { field.reference_to = field.reference_to.filter((ref) => !missingRefs.includes(ref)); fixStatus = 'Fixed'; diff --git a/packages/contentstack-audit/src/modules/entries.ts b/packages/contentstack-audit/src/modules/entries.ts index ccb141ff40..7a820cd2c7 100644 --- a/packages/contentstack-audit/src/modules/entries.ts +++ b/packages/contentstack-audit/src/modules/entries.ts @@ -600,21 +600,31 @@ export default class Entries { fixJsonRteMissingReferences( tree: Record[], field: ReferenceFieldDataType | JsonRTEFieldDataType, - entry: EntryJsonRTEFieldDataType, + entry: EntryJsonRTEFieldDataType | EntryJsonRTEFieldDataType[], ) { - entry.children = entry.children - .map((child) => { - const refExist = this.jsonRefCheck(tree, field, child); + if (Array.isArray(entry)) { + entry = entry.map((child: any, index) => { + return this.fixJsonRteMissingReferences( + [...tree, { index, type: (child as any)?.type, uid: child?.uid }], + field, + child, + ); + }) as EntryJsonRTEFieldDataType[]; + } else { + entry.children = entry.children + .map((child) => { + const refExist = this.jsonRefCheck(tree, field, child); - if (!refExist) return null; + if (!refExist) return null; - if (isEmpty(child.children)) { - child = this.fixJsonRteMissingReferences(tree, field, child); - } + if (isEmpty(child.children)) { + child = this.fixJsonRteMissingReferences(tree, field, child) as EntryJsonRTEFieldDataType; + } - return child; - }) - .filter((val) => val) as EntryJsonRTEFieldDataType[]; + return child; + }) + .filter((val) => val) as EntryJsonRTEFieldDataType[]; + } return entry; } From e6c3643ed8642295bcd2ed63172656e6ae3f022e Mon Sep 17 00:00:00 2001 From: Antony Date: Thu, 12 Oct 2023 19:31:39 +0530 Subject: [PATCH 3/3] Doc: Doc team reviewed text updated --- packages/contentstack-audit/package.json | 2 +- packages/contentstack-audit/src/messages/index.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/contentstack-audit/package.json b/packages/contentstack-audit/package.json index f7d5de36c3..cf059dba52 100644 --- a/packages/contentstack-audit/package.json +++ b/packages/contentstack-audit/package.json @@ -66,7 +66,7 @@ "description": "Perform audits and find possible errors in the exported Contentstack data" }, "cm:stacks:audit:fix": { - "description": "Audit and fix possible errors in the exported data" + "description": "Perform audits and fix possible errors in the exported Contentstack data." } }, "repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-audit/<%- commandPath %>" diff --git a/packages/contentstack-audit/src/messages/index.ts b/packages/contentstack-audit/src/messages/index.ts index 6ade9f6012..a573976c2c 100644 --- a/packages/contentstack-audit/src/messages/index.ts +++ b/packages/contentstack-audit/src/messages/index.ts @@ -24,10 +24,10 @@ const auditMsg = { }; const auditFixMsg = { - COPY_DATA: 'Create backup from original data', + COPY_DATA: 'Create backup from the original data.', BKP_PATH: 'Provide the path to backup the copied data.', - FIXED_CONTENT_PATH_MAG: 'You can locate the fixed content at {path}', - AUDIT_FIX_CMD_DESCRIPTION: 'Audit and fix possible errors in the exported data', + FIXED_CONTENT_PATH_MAG: 'You can locate the fixed content at {path}.', + AUDIT_FIX_CMD_DESCRIPTION: 'Perform audits and fix possible errors in the exported Contentstack data.', }; const messages: typeof errors & typeof commonMsg & typeof auditMsg & typeof auditFixMsg = {