Skip to content

Commit

Permalink
Merge pull request #1322 from contentstack/feat/CS-43978
Browse files Browse the repository at this point in the history
CS-43978- Audit and Audit fix for Extensions
  • Loading branch information
cs-raj authored Mar 8, 2024
2 parents 37b9e07 + f5f3083 commit 2b835fc
Show file tree
Hide file tree
Showing 16 changed files with 1,425 additions and 18 deletions.
82 changes: 72 additions & 10 deletions packages/contentstack-audit/src/audit-base-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import config from './config';
import { print } from './util/log';
import { auditMsg } from './messages';
import { BaseCommand } from './base-command';
import { Entries, GlobalField, ContentType } from './modules';
import { Entries, GlobalField, ContentType, Extensions } from './modules';
import { CommandNames, ContentTypeStruct, OutputColumn, RefErrorReturnType } from './types';

export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseCommand> {
Expand Down Expand Up @@ -42,15 +42,21 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
await this.createBackUp();
this.sharedConfig.reportPath = resolve(this.flags['report-path'] || process.cwd(), 'audit-report');

const { missingCtRefs, missingGfRefs, missingEntryRefs } = await this.scanAndFix();
const { missingCtRefs, missingGfRefs, missingEntryRefs, missingCtRefsInExtensions } = await this.scanAndFix();

this.showOutputOnScreen([
{ module: 'Content types', missingRefs: missingCtRefs },
{ module: 'Global Fields', missingRefs: missingGfRefs },
{ module: 'Entries', missingRefs: missingEntryRefs },
]);

if (!isEmpty(missingCtRefs) || !isEmpty(missingGfRefs) || !isEmpty(missingEntryRefs)) {
this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Extensions', missingRefs: missingCtRefsInExtensions }]);

if (
!isEmpty(missingCtRefs) ||
!isEmpty(missingGfRefs) ||
!isEmpty(missingEntryRefs) ||
!isEmpty(missingCtRefsInExtensions)
) {
if (this.currentCommand === 'cm:stacks:audit') {
this.log(this.$t(auditMsg.FINAL_REPORT_PATH, { path: this.sharedConfig.reportPath }), 'warn');
} else {
Expand All @@ -70,7 +76,12 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
}
}

return !isEmpty(missingCtRefs) || !isEmpty(missingGfRefs) || !isEmpty(missingEntryRefs);
return (
!isEmpty(missingCtRefs) ||
!isEmpty(missingGfRefs) ||
!isEmpty(missingEntryRefs) ||
!isEmpty(missingCtRefsInExtensions)
);
}

/**
Expand All @@ -81,7 +92,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
*/
async scanAndFix() {
let { ctSchema, gfSchema } = this.getCtAndGfSchema();
let missingCtRefs, missingGfRefs, missingEntryRefs;
let missingCtRefs, missingGfRefs, missingEntryRefs, missingCtRefsInExtensions;
for (const module of this.sharedConfig.flags.modules || this.sharedConfig.modules) {
print([
{
Expand Down Expand Up @@ -113,6 +124,10 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
missingEntryRefs = await new Entries(cloneDeep(constructorParam)).run();
await this.prepareReport(module, missingEntryRefs);
break;
case 'extensions':
missingCtRefsInExtensions = await new Extensions(cloneDeep(constructorParam)).run();
await this.prepareReport(module, missingCtRefsInExtensions);
break;
}

print([
Expand All @@ -129,7 +144,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
]);
}

return { missingCtRefs, missingGfRefs, missingEntryRefs };
return { missingCtRefs, missingGfRefs, missingEntryRefs, missingCtRefsInExtensions };
}

/**
Expand Down Expand Up @@ -258,6 +273,52 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
}
}

// Make it generic it takes the column header as param
showOutputOnScreenWorkflowsAndExtension(allMissingRefs: { module: string; missingRefs?: Record<string, any> }[]) {
if (!this.sharedConfig.showTerminalOutput || this.flags['external-config']?.noTerminalOutput) {
return;
}

this.log(''); // Adding a new line

for (const { module, missingRefs } of allMissingRefs) {
if (isEmpty(missingRefs)) {
continue;
}

print([{ bold: true, color: 'cyan', message: ` ${module}` }]);

const tableValues = Object.values(missingRefs).flat();

const tableKeys = Object.keys(missingRefs[0]);
const arrayOfObjects = tableKeys.map((key) => {
if (['title', 'name', 'uid', 'content_types', 'fixStatus'].includes(key)) {
return {
[key]: {
minWidth: 7,
header: key,
get: (row: Record<string, unknown>) => {
if(key==='fixStatus') {
return chalk.green(typeof row[key] === 'object' ? JSON.stringify(row[key]) : row[key]);
} else if(key==='content_types') {
return chalk.red(typeof row[key] === 'object' ? JSON.stringify(row[key]) : row[key]);
} else {
return chalk.white(typeof row[key] === 'object' ? JSON.stringify(row[key]) : row[key]);
}
},
},
};
}
return {};
});

const mergedObject = Object.assign({}, ...arrayOfObjects);

ux.table(tableValues, mergedObject, { ...this.flags });
this.log(''); // Adding a new line
}
}

/**
* The function prepares a report by writing a JSON file and a CSV file with a list of missing
* references for a given module.
Expand Down Expand Up @@ -311,13 +372,14 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
}

const rowData: Record<string, string | string[]>[] = [];

for (const issue of missingRefs) {
let row: Record<string, string | string[]> = {};

for (const column of columns) {
row[column] = issue[OutputColumn[column]];
row[column] = typeof row[column] === 'object' ? JSON.stringify(row[column]) : row[column];
if (Object.keys(issue).includes(OutputColumn[column])) {
row[column] = issue[OutputColumn[column]] as string;
row[column] = typeof row[column] === 'object' ? JSON.stringify(row[column]) : row[column];
}
}

if (this.currentCommand === 'cm:stacks:audit:fix') {
Expand Down
7 changes: 6 additions & 1 deletion packages/contentstack-audit/src/config/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ const config = {
showTerminalOutput: true,
skipRefs: ['sys_assets'],
skipFieldTypes: ['taxonomy', 'group'],
modules: ['content-types', 'global-fields', 'entries'],
modules: ['content-types', 'global-fields', 'entries', 'extensions'],
'fix-fields': ['reference', 'global_field', 'json:rte', 'json:extension', 'blocks', 'group'],
moduleConfig: {
'content-types': {
Expand All @@ -25,6 +25,11 @@ const config = {
dirName: 'locales',
fileName: 'locales.json',
},
extensions: {
name: 'extensions',
dirName: 'extensions',
fileName: 'extensions.json',
},
},
entries: {
systemKeys: [
Expand Down
3 changes: 3 additions & 0 deletions packages/contentstack-audit/src/messages/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ const commonMsg = {
CONFIG: 'Path of the external config',
DATA_DIR: 'Path where the data is stored',
FIX_CONFIRMATION: 'Would you like to overwrite existing file.?',
EXTENSION_FIX_WARN: `The extension associated with UID {uid} and title '{title}' will be removed.`,
EXTENSION_FIX_CONFIRMATION: `Would you like to overwrite existing file?`,
};

const auditMsg = {
Expand All @@ -28,6 +30,7 @@ const auditMsg = {
FINAL_REPORT_PATH: "Reports ready. Please find the reports at '{path}'.",
SCAN_CT_SUCCESS_MSG: "Successfully completed the scanning of {module} '{title}'.",
SCAN_ENTRY_SUCCESS_MSG: "Successfully completed the scanning of {module} ({local}) '{title}'.",
SCAN_EXT_SUCCESS_MSG: "Successfully completed scanning the {module} titled '{title}' with UID '{uid}'",
AUDIT_CMD_DESCRIPTION: 'Perform audits and find possible errors in the exported Contentstack data',
};

Expand Down
120 changes: 120 additions & 0 deletions packages/contentstack-audit/src/modules/extensions.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
import path, { join, resolve } from 'path';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { cloneDeep } from 'lodash';
import { LogFn, ConfigType, ContentTypeStruct, CtConstructorParam, ModuleConstructorParam, Extension } from '../types';
import { ux } from '@contentstack/cli-utilities';

import auditConfig from '../config';
import { $t, auditMsg, commonMsg } from '../messages';
import { values } from 'lodash';

export default class Extensions {
public log: LogFn;
protected fix: boolean;
public fileName: any;
public config: ConfigType;
public folderPath: string;
public extensionsSchema: Extension[];
public ctSchema: ContentTypeStruct[];
public moduleName: keyof typeof auditConfig.moduleConfig;
public ctUidSet: Set<string>;
public missingCtInExtensions: Extension[];
public missingCts: Set<string>;
public extensionsPath: string;

constructor({
log,
fix,
config,
moduleName,
ctSchema,
}: ModuleConstructorParam & Pick<CtConstructorParam, 'ctSchema'>) {
this.log = log;
this.config = config;
this.fix = fix ?? false;
this.ctSchema = ctSchema;
this.extensionsSchema = [];
this.moduleName = moduleName ?? 'extensions';
this.fileName = config.moduleConfig[this.moduleName].fileName;
this.folderPath = resolve(config.basePath, config.moduleConfig[this.moduleName].dirName);
this.ctUidSet = new Set(['$all']);
this.missingCtInExtensions = [];
this.missingCts = new Set();
this.extensionsPath = '';
}

async run() {
if (!existsSync(this.folderPath)) {
this.log(`Skipping ${this.moduleName} audit`, 'warn');
this.log($t(auditMsg.NOT_VALID_PATH, { path: this.folderPath }), { color: 'yellow' });
return {};
}

this.extensionsPath = path.join(this.folderPath, this.fileName);

this.extensionsSchema = existsSync(this.extensionsPath)
? values(JSON.parse(readFileSync(this.extensionsPath, 'utf-8')) as Extension[])
: [];
this.ctSchema.map((ct) => this.ctUidSet.add(ct.uid));
for (const ext of this.extensionsSchema) {
const { title, uid, scope } = ext;
const ctNotPresent = scope?.content_types.filter((ct) => !this.ctUidSet.has(ct));

if (ctNotPresent?.length && ext.scope) {
ext.content_types = ctNotPresent;
ctNotPresent.forEach((ct) => this.missingCts.add(ct));
this.missingCtInExtensions.push(cloneDeep(ext));
}

this.log(
$t(auditMsg.SCAN_EXT_SUCCESS_MSG, {
title,
module: this.config.moduleConfig[this.moduleName].name,
uid,
}),
'info',
);
}

if (this.fix && this.missingCtInExtensions.length) {
await this.fixExtensionsScope(cloneDeep(this.missingCtInExtensions));
this.missingCtInExtensions.forEach((ext) => (ext.fixStatus = 'Fixed'));
return this.missingCtInExtensions
}
return this.missingCtInExtensions;
}

async fixExtensionsScope(missingCtInExtensions: Extension[]) {
let newExtensionSchema: Record<string, Extension> = existsSync(this.extensionsPath)
? JSON.parse(readFileSync(this.extensionsPath, 'utf8'))
: {};
for (const ext of missingCtInExtensions) {
const { uid, title } = ext;
const fixedCts = ext?.scope?.content_types.filter((ct) => !this.missingCts.has(ct));
if (fixedCts?.length && newExtensionSchema[uid]?.scope) {
newExtensionSchema[uid].scope.content_types = fixedCts;
} else {
this.log($t(commonMsg.EXTENSION_FIX_WARN, { title: title, uid }), { color: 'yellow' });
const shouldDelete = this.config.flags.yes || (await ux.confirm(commonMsg.EXTENSION_FIX_CONFIRMATION));
if (shouldDelete) {
delete newExtensionSchema[uid];
}
}
}
await this.writeFixContent(newExtensionSchema);
}

async writeFixContent(fixedExtensions: Record<string, Extension>) {
if (
this.fix ||
(this.config.flags['copy-dir'] ||
this.config.flags['external-config']?.skipConfirm ||
(await ux.confirm(commonMsg.FIX_CONFIRMATION)))
) {
writeFileSync(
join(this.folderPath, this.config.moduleConfig[this.moduleName].fileName),
JSON.stringify(fixedExtensions),
);
}
}
}
10 changes: 5 additions & 5 deletions packages/contentstack-audit/src/modules/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import Entries from "./entries"
import GlobalField from "./global-fields"
import ContentType from "./content-types"

export { Entries, GlobalField, ContentType }
import Entries from './entries';
import GlobalField from './global-fields';
import ContentType from './content-types';
import Extensions from './extensions';
export { Entries, GlobalField, ContentType, Extensions };
6 changes: 6 additions & 0 deletions packages/contentstack-audit/src/types/content-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ type RefErrorReturnType = {
missingRefs: string[];
display_name: string;
tree: Record<string, unknown>[];
uid?: string;
content_types?: string[];
title?: string;
};

// NOTE Type 1
Expand Down Expand Up @@ -113,6 +116,9 @@ enum OutputColumn {
'Field type' = 'data_type',
'Missing references' = 'missingRefs',
Path = 'treeStr',
title = 'title',
'uid' = 'uid',
'missingCts' = 'content_types',
}

export {
Expand Down
24 changes: 24 additions & 0 deletions packages/contentstack-audit/src/types/extensions.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
export interface Extension {
stackHeaders: {
api_key: string;
};
urlPath: string;
uid: string;
created_at: string;
updated_at: string;
created_by: string;
updated_by: string;
tags?: [];
_version: number;
title: string;
config: {};
type: 'field';
data_type: string;
multiple: boolean;
srcdoc?: string;
scope: {
content_types: string[];
};
content_types?: string[];
fixStatus?: string;
}
1 change: 1 addition & 0 deletions packages/contentstack-audit/src/types/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ export * from './utils';
export * from './common';
export * from './entries';
export * from './content-types';
export * from './extensions';
Loading

0 comments on commit 2b835fc

Please sign in to comment.