Skip to content

Commit

Permalink
Merge pull request #1399 from contentstack/feat/DX-523
Browse files Browse the repository at this point in the history 
DX - 523 - Audit and Audit Fix for Mandatory Fields
  • Loading branch information
@cs-raj
cs-raj authored May 7, 2024
2 parents 905a28d + 6495f46 commit f20c73d
Show file tree
Hide file tree
Showing 6 changed files with 152 additions and 22 deletions.
30 changes: 21 additions & 9 deletions packages/contentstack-audit/src/audit-base-command.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
missingCtRefsInExtensions,
missingCtRefsInWorkflow,
missingSelectFeild,
missingMandatoryFields,
} = await this.scanAndFix();

this.showOutputOnScreen([
Expand All @@ -65,7 +66,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Extensions', missingRefs: missingCtRefsInExtensions }]);
this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Workflows', missingRefs: missingCtRefsInWorkflow }]);
this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Entries Select Field', missingRefs: missingSelectFeild }]);

this.showOutputOnScreenWorkflowsAndExtension([
{ module: 'Entries Mandatory Field', missingRefs: missingMandatoryFields },
]);
if (
!isEmpty(missingCtRefs) ||
!isEmpty(missingGfRefs) ||
Expand Down Expand Up @@ -117,7 +120,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
missingCtRefsInExtensions,
missingCtRefsInWorkflow,
missingSelectFeild,
missingEntry;
missingEntry,
missingMandatoryFields;

for (const module of this.sharedConfig.flags.modules || this.sharedConfig.modules) {
print([
{
Expand Down Expand Up @@ -148,9 +153,12 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
missingEntry = await new Entries(cloneDeep(constructorParam)).run();
missingEntryRefs = missingEntry.missingEntryRefs ?? {};
missingSelectFeild = missingEntry.missingSelectFeild ?? {};
missingMandatoryFields = missingEntry.missingMandatoryFields ?? {};
await this.prepareReport(module, missingEntryRefs);

await this.prepareReport(`entries_Select_feild`, missingSelectFeild);
await this.prepareReport(`Entries_Select_feild`, missingSelectFeild);

await this.prepareReport('Entries_Mandatory_feild', missingMandatoryFields);
break;
case 'workflows':
missingCtRefsInWorkflow = await new Workflows({
Expand Down Expand Up @@ -189,6 +197,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
missingCtRefsInExtensions,
missingCtRefsInWorkflow,
missingSelectFeild,
missingMandatoryFields,
};
}

Expand Down Expand Up @@ -335,17 +344,20 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
missingRefs = Object.values(missingRefs).flat();
const tableKeys = Object.keys(missingRefs[0]);
const arrayOfObjects = tableKeys.map((key) => {
if (
config.OutputTableKeys.includes(key)
) {
if (config.OutputTableKeys.includes(key)) {
return {
[key]: {
minWidth: 7,
header: key,
get: (row: Record<string, unknown>) => {
if (key === 'fixStatus') {
return chalk.green(typeof row[key] === 'object' ? JSON.stringify(row[key]) : row[key]);
} else if (key === 'content_types' || key === 'branches' || key === 'missingCTSelectFieldValues') {
} else if (
key === 'content_types' ||
key === 'branches' ||
key === 'missingCTSelectFieldValues' ||
key === 'missingFieldUid'
) {
return chalk.red(typeof row[key] === 'object' ? JSON.stringify(row[key]) : row[key]);
} else {
return chalk.white(typeof row[key] === 'object' ? JSON.stringify(row[key]) : row[key]);
Expand Down Expand Up @@ -374,7 +386,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
* @returns The function `prepareReport` returns a Promise that resolves to `void`.
*/
prepareReport(
moduleName: keyof typeof config.moduleConfig | 'entries_Select_feild',
moduleName: keyof typeof config.moduleConfig | keyof typeof config.ReportTitleForEntries,
listOfMissingRefs: Record<string, any>,
): Promise<void> {
if (isEmpty(listOfMissingRefs)) return Promise.resolve(void 0);
Expand All @@ -401,7 +413,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
* @returns The function `prepareCSV` returns a Promise that resolves to `void`.
*/
prepareCSV(
moduleName: keyof typeof config.moduleConfig | 'entries_Select_feild',
moduleName: keyof typeof config.moduleConfig | keyof typeof config.ReportTitleForEntries,
listOfMissingRefs: Record<string, any>,
): Promise<void> {
const csvPath = join(this.sharedConfig.reportPath, `${moduleName}.csv`);
Expand Down
10 changes: 8 additions & 2 deletions packages/contentstack-audit/src/config/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ const config = {
],
},
//These keys will be used output the modules with issues and fixes on console
OutputTableKeys : [
OutputTableKeys: [
'title',
'name',
'uid',
Expand All @@ -69,7 +69,13 @@ const config = {
'treeStr',
'missingCTSelectFieldValues',
'min_instance',
]
'missingFieldUid',
'isPublished',
],
ReportTitleForEntries: {
Entries_Select_feild: 'Entries_Select_feild',
Entries_Mandatory_feild: 'Entries_Mandatory_feild',
},
};

export default config;
2 changes: 2 additions & 0 deletions packages/contentstack-audit/src/messages/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ const auditFixMsg = {
EMPTY_FIX_MSG: 'Successfully removed the empty field/block found at {path} from the schema.',
AUDIT_FIX_CMD_DESCRIPTION: 'Perform audits and fix possible errors in the exported Contentstack data.',
WF_FIX_MSG: 'Successfully removed the workflow {uid} named {name}.',
ENTRY_MANDATORY_FIELD_FIX: `Removing the publish details from entry uid '{uid}' from locale '{locale}'`,
ENTRY_SELECT_FIELD_FIX: `Adding the value '{value}' in select field of uid '{uid}'`
};

const messages: typeof errors &
Expand Down
125 changes: 115 additions & 10 deletions packages/contentstack-audit/src/modules/entries.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { existsSync, readFileSync, writeFileSync } from 'fs';

import auditConfig from '../config';
import ContentType from './content-types';
import { $t, auditMsg, commonMsg } from '../messages';
import { $t, auditFixMsg, auditMsg, commonMsg } from '../messages';
import {
LogFn,
Locale,
Expand Down Expand Up @@ -56,6 +56,7 @@ export default class Entries {
protected entries!: Record<string, EntryStruct>;
protected missingRefs: Record<string, any> = {};
protected missingSelectFeild: Record<string, any> = {};
protected missingMandatoryFields: Record<string, any> = {};
public entryMetaData: Record<string, any>[] = [];
public moduleName: keyof typeof auditConfig.moduleConfig = 'entries';
public isEntryWithoutTitleField: boolean = false;
Expand Down Expand Up @@ -109,11 +110,35 @@ export default class Entries {
if (!this.missingSelectFeild[this.currentUid]) {
this.missingSelectFeild[this.currentUid] = [];
}

if (!this.missingMandatoryFields[this.currentUid]) {
this.missingMandatoryFields[this.currentUid] = [];
}
if (this.fix) {
this.removeMissingKeysOnEntry(ctSchema.schema as ContentTypeSchemaType[], this.entries[entryUid]);
}

this.lookForReference([{ locale: code, uid, name: title }], ctSchema, this.entries[entryUid]);

const fields = this.missingMandatoryFields[uid];
const isPublished = entry.publish_details.length > 0;
if ((this.fix && fields && isPublished) || (!this.fix && fields)) {
const fixStatus = this.fix ? 'Fixed' : '';
fields?.forEach((field: { isPublished: boolean; fixStatus?: string }) => {
field.isPublished = isPublished;
if (this.fix && isPublished) {
field.fixStatus = fixStatus;
}
});

if (this.fix && isPublished) {
this.log($t(auditFixMsg.ENTRY_MANDATORY_FIELD_FIX, { uid, locale: code }), 'error');
entry.publish_details = [];
}
} else {
delete this.missingMandatoryFields[uid];
}

const message = $t(auditMsg.SCAN_ENTRY_SUCCESS_MSG, {
title,
local: code,
Expand All @@ -132,7 +157,12 @@ export default class Entries {
// this.log('', 'info'); // Adding empty line

this.removeEmptyVal();
return { missingEntryRefs: this.missingRefs, missingSelectFeild: this.missingSelectFeild };

return {
missingEntryRefs: this.missingRefs,
missingSelectFeild: this.missingSelectFeild,
missingMandatoryFields: this.missingMandatoryFields,
};
}

/**
Expand All @@ -149,6 +179,11 @@ export default class Entries {
delete this.missingSelectFeild[propName];
}
}
for (let propName in this.missingMandatoryFields) {
if (!this.missingMandatoryFields[propName].length) {
delete this.missingMandatoryFields[propName];
}
}
}

/**
Expand Down Expand Up @@ -237,7 +272,16 @@ export default class Entries {

for (const child of field?.schema ?? []) {
const { uid } = child;
if (!entry?.[uid]) continue;
this.missingMandatoryFields[this.currentUid].push(
...this.validateMandatoryFields(
[...tree, { uid: field.uid, name: child.display_name, field: uid }],
child,
entry,
),
);
if (!entry?.[uid] && !child.hasOwnProperty('display_type')) {
continue;
}

switch (child.data_type) {
case 'reference':
Expand Down Expand Up @@ -668,7 +712,24 @@ export default class Entries {
*/
validateSelectField(tree: Record<string, unknown>[], fieldStructure: SelectFeildStruct, field: any) {
const { display_name, enum: selectOptions, multiple, min_instance, display_type } = fieldStructure;

if (field === null || field === '' || field?.length === 0 || !field) {
let missingCTSelectFieldValues = 'Not Selected';
return [
{
uid: this.currentUid,
name: this.currentTitle,
display_name,
display_type,
missingCTSelectFieldValues,
min_instance: min_instance ?? 'NA',
tree,
treeStr: tree
.map(({ name }) => name)
.filter((val) => val)
.join(' ➜ '),
},
];
}
let missingCTSelectFieldValues;

if (multiple) {
Expand Down Expand Up @@ -714,7 +775,7 @@ export default class Entries {
* @returns
*/
fixSelectField(tree: Record<string, unknown>[], field: SelectFeildStruct, entry: any) {
const { enum: selectOptions, multiple, min_instance, display_type, display_name } = field;
const { enum: selectOptions, multiple, min_instance, display_type, display_name, uid } = field;

let missingCTSelectFieldValues;
let isMissingValuePresent = false;
Expand All @@ -724,7 +785,7 @@ export default class Entries {
let { notPresent, filteredFeild } = obj;
entry = filteredFeild;
missingCTSelectFieldValues = notPresent;
if(missingCTSelectFieldValues.length) {
if (missingCTSelectFieldValues.length) {
isMissingValuePresent = true;
}
if (min_instance && Array.isArray(entry)) {
Expand All @@ -736,20 +797,24 @@ export default class Entries {
.slice(0, missingInstances)
.map((choice) => choice.value);
entry.push(...newValues);
this.log($t(auditFixMsg.ENTRY_SELECT_FIELD_FIX, { value: newValues.join(' '), uid }), 'error');
}
} else {
if (entry.length === 0) {
isMissingValuePresent = true;
const defaultValue = selectOptions.choices.length > 0 ? selectOptions.choices[0].value : null;
entry.push(defaultValue);
this.log($t(auditFixMsg.ENTRY_SELECT_FIELD_FIX, { value: defaultValue as string, uid }), 'error');
}
}
} else {
const isPresent = selectOptions.choices.some((choice) => choice.value === entry);
if (!isPresent) {
missingCTSelectFieldValues = entry;
isMissingValuePresent = true;
entry = selectOptions.choices.length > 0 ? selectOptions.choices[0].value : null;
let defaultValue = selectOptions.choices.length > 0 ? selectOptions.choices[0].value : null;
entry = defaultValue;
this.log($t(auditFixMsg.ENTRY_SELECT_FIELD_FIX, { value: defaultValue as string, uid }), 'error');
}
}
if (display_type && isMissingValuePresent) {
Expand All @@ -770,6 +835,46 @@ export default class Entries {
}
return entry;
}

validateMandatoryFields(tree: Record<string, unknown>[], fieldStructure: any, entry: any) {
const { display_name, multiple, data_type, mandatory, field_metadata, uid } = fieldStructure;

const isJsonRteEmpty = () => {
const jsonNode = multiple
? entry[uid]?.[0]?.children?.[0]?.children?.[0]?.text
: entry[uid]?.children?.[0]?.children?.[0]?.text;
return jsonNode === '';
};

const isEntryEmpty = () => {
const fieldValue = multiple ? entry[uid]?.length : entry[uid];
return fieldValue === '' || !fieldValue;
};

if (mandatory) {
if (
(data_type === 'json' && field_metadata.allow_json_rte && isJsonRteEmpty()) ||
(!(data_type === 'json') && isEntryEmpty())
) {
return [
{
uid: this.currentUid,
name: this.currentTitle,
display_name,
missingFieldUid: uid,
tree,
treeStr: tree
.filter(({ name }) => name)
.map(({ name }) => name)
.join(' ➜ '),
},
];
}
}

return [];
}

/**
*
* @param field It contains the value to be searched
Expand All @@ -779,10 +884,10 @@ export default class Entries {
findNotPresentSelectField(field: any, selectOptions: any) {
let present = [];
let notPresent = [];
const choicesMap = new Map(selectOptions.choices.map((choice: { value: any; }) => [choice.value, choice]));
const choicesMap = new Map(selectOptions.choices.map((choice: { value: any }) => [choice.value, choice]));
for (const value of field) {
const choice:any = choicesMap.get(value);
const choice: any = choicesMap.get(value);

if (choice) {
present.push(choice.value);
} else {
Expand Down
6 changes: 5 additions & 1 deletion packages/contentstack-audit/src/types/content-types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ type ContentTypeStruct = {
title: string;
description: string;
schema?: ContentTypeSchemaType[];
mandatory: boolean;
};

type ModuleConstructorParam = {
Expand All @@ -38,6 +39,7 @@ type CommonDataTypeStruct = {
ref_multiple: boolean;
allow_json_rte: boolean;
} & AnyProperty;
mandatory: boolean
};

type RefErrorReturnType = {
Expand Down Expand Up @@ -138,7 +140,9 @@ enum OutputColumn {
'missingCts' = 'content_types',
'Missing Branches' = 'branches',
'MissingValues' = 'missingCTSelectFieldValues',
"Minimum Required Instaces" = 'min_instance'
'Minimum Required Instaces' = 'min_instance',
'missingFieldUid' = 'missingFieldUid',
'isPublished' = 'isPublished'
}

export {
Expand Down
1 change: 1 addition & 0 deletions packages/contentstack-audit/src/types/entries.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ type Locale = {
type EntryStruct = {
uid: string;
title: string;
publish_details: []
} & {
[key: string]:
| EntryReferenceFieldDataType[]
Expand Down

0 comments on commit f20c73d

Please sign in to comment.