Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DX - 94 - SRE Fixes #1415

Merged
merged 12 commits into from
May 21, 2024
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
634 changes: 390 additions & 244 deletions package-lock.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion packages/contentstack-audit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ $ npm install -g @contentstack/cli-audit
$ csdx COMMAND
running command...
$ csdx (--version|-v)
@contentstack/cli-audit/1.6.1 darwin-arm64 node-v21.6.2
@contentstack/cli-audit/1.6.2 darwin-arm64 node-v21.6.2
$ csdx --help [COMMAND]
USAGE
$ csdx COMMAND
Expand Down
6 changes: 3 additions & 3 deletions packages/contentstack-audit/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@contentstack/cli-audit",
"version": "1.6.1",
"version": "1.6.2",
"description": "Contentstack audit plugin",
"author": "Contentstack CLI",
"homepage": "https://github.com/contentstack/cli",
Expand All @@ -19,7 +19,7 @@
],
"dependencies": {
"@contentstack/cli-command": "~1.2.18",
"@contentstack/cli-utilities": "~1.6.1",
"@contentstack/cli-utilities": "~1.6.2",
"@oclif/plugin-help": "^5",
"@oclif/plugin-plugins": "^5.0.0",
"chalk": "^4.1.2",
Expand Down Expand Up @@ -76,7 +76,7 @@
"prepack": "npm run build && oclif manifest && oclif readme",
"test": "mocha --forbid-only \"test/**/*.test.ts\"",
"version": "oclif readme && git add README.md",
"clean": "rm -rf ./lib ./node_modules tsconfig.tsbuildinfo oclif.manifest.json",
"clean": "rm -rf ./lib tsconfig.tsbuildinfo oclif.manifest.json",
cs-raj marked this conversation as resolved.
Show resolved Hide resolved
"test:unit:report": "nyc --extension .ts mocha --forbid-only \"test/unit/**/*.test.ts\""
},
"engines": {
Expand Down
93 changes: 49 additions & 44 deletions packages/contentstack-audit/src/audit-base-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { v4 as uuid } from 'uuid';
import isEmpty from 'lodash/isEmpty';
import { join, resolve } from 'path';
import cloneDeep from 'lodash/cloneDeep';
import { cliux, ux } from '@contentstack/cli-utilities';
import { cliux, sanitizePath, ux } from '@contentstack/cli-utilities';
import { createWriteStream, existsSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'fs';

import config from './config';
Expand Down Expand Up @@ -396,7 +396,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
}

// NOTE write int json
writeFileSync(join(this.sharedConfig.reportPath, `${moduleName}.json`), JSON.stringify(listOfMissingRefs));
writeFileSync(join(sanitizePath(this.sharedConfig.reportPath), `${sanitizePath(moduleName)}.json`), JSON.stringify(listOfMissingRefs));
cs-raj marked this conversation as resolved.
Show resolved Hide resolved

// NOTE write into CSV
return this.prepareCSV(moduleName, listOfMissingRefs);
Expand All @@ -416,50 +416,55 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
moduleName: keyof typeof config.moduleConfig | keyof typeof config.ReportTitleForEntries,
listOfMissingRefs: Record<string, any>,
): Promise<void> {
const csvPath = join(this.sharedConfig.reportPath, `${moduleName}.csv`);

return new Promise<void>((resolve, reject) => {
// file deepcode ignore MissingClose: Will auto close once csv stream end
const ws = createWriteStream(csvPath).on('error', reject);
const defaultColumns = Object.keys(OutputColumn);
const userDefinedColumns = this.sharedConfig.flags.columns ? this.sharedConfig.flags.columns.split(',') : null;
let missingRefs: RefErrorReturnType[] | WorkflowExtensionsRefErrorReturnType[] =
Object.values(listOfMissingRefs).flat();
const columns: (keyof typeof OutputColumn)[] = userDefinedColumns
? [...userDefinedColumns, ...defaultColumns.filter((val: string) => !userDefinedColumns.includes(val))]
: defaultColumns;

if (this.sharedConfig.flags.filter) {
const [column, value]: [keyof typeof OutputColumn, string] = this.sharedConfig.flags.filter.split('=');
// Filter the missingRefs array
missingRefs = missingRefs.filter((row) => {
if (OutputColumn[column] in row) {
const rowKey = OutputColumn[column] as keyof (RefErrorReturnType | WorkflowExtensionsRefErrorReturnType);
return row[rowKey] === value;
if(Object.keys(config.moduleConfig).includes(moduleName)){
const csvPath = join(sanitizePath(this.sharedConfig.reportPath), `${sanitizePath(moduleName)}.csv`);
return new Promise<void>((resolve, reject) => {
// file deepcode ignore MissingClose: Will auto close once csv stream end
const ws = createWriteStream(csvPath).on('error', reject);
const defaultColumns = Object.keys(OutputColumn);
const userDefinedColumns = this.sharedConfig.flags.columns ? this.sharedConfig.flags.columns.split(',') : null;
let missingRefs: RefErrorReturnType[] | WorkflowExtensionsRefErrorReturnType[] =
Object.values(listOfMissingRefs).flat();
const columns: (keyof typeof OutputColumn)[] = userDefinedColumns
? [...userDefinedColumns, ...defaultColumns.filter((val: string) => !userDefinedColumns.includes(val))]
: defaultColumns;

if (this.sharedConfig.flags.filter) {
const [column, value]: [keyof typeof OutputColumn, string] = this.sharedConfig.flags.filter.split('=');
// Filter the missingRefs array
missingRefs = missingRefs.filter((row) => {
if (OutputColumn[column] in row) {
const rowKey = OutputColumn[column] as keyof (RefErrorReturnType | WorkflowExtensionsRefErrorReturnType);
return row[rowKey] === value;
}
return false;
});
}

const rowData: Record<string, string | string[]>[] = [];
for (const issue of missingRefs) {
let row: Record<string, string | string[]> = {};

for (const column of columns) {
if (Object.keys(issue).includes(OutputColumn[column])) {
const issueKey = OutputColumn[column] as keyof typeof issue;
row[column] = issue[issueKey] as string;
row[column] = typeof row[column] === 'object' ? JSON.stringify(row[column]) : row[column];
}
}
return false;
});
}

const rowData: Record<string, string | string[]>[] = [];
for (const issue of missingRefs) {
let row: Record<string, string | string[]> = {};

for (const column of columns) {
if (Object.keys(issue).includes(OutputColumn[column])) {
const issueKey = OutputColumn[column] as keyof typeof issue;
row[column] = issue[issueKey] as string;
row[column] = typeof row[column] === 'object' ? JSON.stringify(row[column]) : row[column];

if (this.currentCommand === 'cm:stacks:audit:fix') {
row['Fix status'] = row.fixStatus;
}

rowData.push(row);
}

if (this.currentCommand === 'cm:stacks:audit:fix') {
row['Fix status'] = row.fixStatus;
}

rowData.push(row);
}
csv.write(rowData, { headers: true }).pipe(ws).on('error', reject).on('finish', resolve);
});
csv.write(rowData, { headers: true }).pipe(ws).on('error', reject).on('finish', resolve);
});
} else {
return new Promise<void>((reject)=>{
return reject()
})
}
}
}
11 changes: 9 additions & 2 deletions packages/contentstack-audit/src/messages/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import memoize from 'lodash/memoize';
import { escapeRegExp } from '@contentstack/cli-utilities';
import { escapeRegExp, validateRegex } from '@contentstack/cli-utilities';

const errors = {};

Expand Down Expand Up @@ -77,7 +77,14 @@ function $t(msg: string, args: Record<string, string>): string {

for (const key of Object.keys(args)) {
const escapedKey = escapeRegExp(key);
msg = msg.replace(new RegExp(`{${escapedKey}}`, 'g'), escapeRegExp(args[key]) || escapedKey);
const escapedKeyRegex = new RegExp(`{${escapedKey}}`, 'g');
let { status } = validateRegex(escapedKeyRegex)
if (status === 'safe') {
const sanitizedValue = args[key] ? escapeRegExp(args[key]) : '';
msg = msg.replace(escapedKeyRegex, sanitizedValue || escapedKey);
} else {

}
cs-raj marked this conversation as resolved.
Show resolved Hide resolved
}

return msg;
Expand Down
12 changes: 9 additions & 3 deletions packages/contentstack-audit/src/modules/content-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import isEmpty from 'lodash/isEmpty';
import { join, resolve } from 'path';
import { existsSync, readFileSync, writeFileSync } from 'fs';

import { ux } from '@contentstack/cli-utilities';
import { sanitizePath, ux } from '@contentstack/cli-utilities';

import {
LogFn,
Expand Down Expand Up @@ -51,11 +51,17 @@ export default class ContentType {
this.fix = fix ?? false;
this.ctSchema = ctSchema;
this.gfSchema = gfSchema;
this.moduleName = moduleName ?? 'content-types';
this.moduleName = this.validateModules(moduleName!,this.config.moduleConfig);
this.fileName = config.moduleConfig[this.moduleName].fileName;
this.folderPath = resolve(config.basePath, config.moduleConfig[this.moduleName].dirName);
this.folderPath = resolve(sanitizePath(config.basePath), sanitizePath(config.moduleConfig[this.moduleName].dirName));
}

validateModules(moduleName:keyof typeof auditConfig.moduleConfig, moduleConfig: Record<string, unknown>):keyof typeof auditConfig.moduleConfig {
if(Object.keys(moduleConfig).includes(moduleName)){
return moduleName;
}
return 'content-types'
}
cs-raj marked this conversation as resolved.
Show resolved Hide resolved
/**
* The `run` function checks if a folder path exists, sets the schema based on the module name,
* iterates over the schema and looks for references, and returns a list of missing references.
Expand Down
73 changes: 40 additions & 33 deletions packages/contentstack-audit/src/modules/entries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import find from 'lodash/find';
import values from 'lodash/values';
import isEmpty from 'lodash/isEmpty';
import { join, resolve } from 'path';
import { ux, FsUtility } from '@contentstack/cli-utilities';
import { ux, FsUtility, sanitizePath } from '@contentstack/cli-utilities';
import { existsSync, readFileSync, writeFileSync } from 'fs';

import auditConfig from '../config';
Expand Down Expand Up @@ -67,9 +67,16 @@ export default class Entries {
this.fix = fix ?? false;
this.ctSchema = ctSchema;
this.gfSchema = gfSchema;
this.moduleName = moduleName ?? 'entries';
this.moduleName = this.validateModules(moduleName!, this.config.moduleConfig);
this.fileName = config.moduleConfig[this.moduleName].fileName;
this.folderPath = resolve(config.basePath, config.moduleConfig.entries.dirName);
this.folderPath = resolve(sanitizePath(config.basePath), sanitizePath(config.moduleConfig.entries.dirName));
}

validateModules(moduleName: keyof typeof auditConfig.moduleConfig, moduleConfig: Record<string, unknown>): keyof typeof auditConfig.moduleConfig {
if (Object.keys(moduleConfig).includes(moduleName)) {
cs-raj marked this conversation as resolved.
Show resolved Hide resolved
return moduleName;
}
return 'entries'
}

/**
Expand Down Expand Up @@ -193,15 +200,15 @@ export default class Entries {
async fixPrerequisiteData() {
this.ctSchema = (await new ContentType({
fix: true,
log: () => {},
log: () => { },
config: this.config,
moduleName: 'content-types',
ctSchema: this.ctSchema,
gfSchema: this.gfSchema,
}).run(true)) as ContentTypeStruct[];
this.gfSchema = (await new GlobalField({
fix: true,
log: () => {},
log: () => { },
config: this.config,
moduleName: 'global-fields',
ctSchema: this.ctSchema,
Expand All @@ -214,7 +221,7 @@ export default class Entries {
if (existsSync(extensionPath)) {
try {
this.extensions = Object.keys(JSON.parse(readFileSync(extensionPath, 'utf8')));
} catch (error) {}
} catch (error) { }
}

if (existsSync(marketplacePath)) {
Expand All @@ -227,7 +234,7 @@ export default class Entries {
) as string[];
this.extensions.push(...metaData);
}
} catch (error) {}
} catch (error) { }
}
}

Expand Down Expand Up @@ -409,19 +416,19 @@ export default class Entries {

return missingRefs.length
? [
{
tree,
data_type,
missingRefs,
display_name,
ct_uid: this.currentUid,
name: this.currentTitle,
treeStr: tree
.map(({ name }) => name)
.filter((val) => val)
.join(' ➜ '),
},
]
{
tree,
data_type,
missingRefs,
display_name,
ct_uid: this.currentUid,
name: this.currentTitle,
treeStr: tree
.map(({ name }) => name)
.filter((val) => val)
.join(' ➜ '),
},
]
: [];
}

Expand Down Expand Up @@ -581,19 +588,19 @@ export default class Entries {

return missingRefs.length
? [
{
tree,
data_type,
missingRefs,
display_name,
uid: this.currentUid,
name: this.currentTitle,
treeStr: tree
.map(({ name }) => name)
.filter((val) => val)
.join(' ➜ '),
},
]
{
tree,
data_type,
missingRefs,
display_name,
uid: this.currentUid,
name: this.currentTitle,
treeStr: tree
.map(({ name }) => name)
.filter((val) => val)
.join(' ➜ '),
},
]
: [];
}

Expand Down
12 changes: 9 additions & 3 deletions packages/contentstack-audit/src/modules/extensions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import path, { join, resolve } from 'path';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { cloneDeep } from 'lodash';
import { LogFn, ConfigType, ContentTypeStruct, CtConstructorParam, ModuleConstructorParam, Extension } from '../types';
import { ux } from '@contentstack/cli-utilities';
import { ux, sanitizePath } from '@contentstack/cli-utilities';

import auditConfig from '../config';
import { $t, auditMsg, commonMsg } from '../messages';
Expand Down Expand Up @@ -34,14 +34,20 @@ export default class Extensions {
this.fix = fix ?? false;
this.ctSchema = ctSchema;
this.extensionsSchema = [];
this.moduleName = moduleName ?? 'extensions';
this.moduleName = this.validateModules(moduleName!,this.config.moduleConfig);
this.fileName = config.moduleConfig[this.moduleName].fileName;
this.folderPath = resolve(config.basePath, config.moduleConfig[this.moduleName].dirName);
this.folderPath = resolve(sanitizePath(config.basePath), sanitizePath(config.moduleConfig[this.moduleName].dirName));
this.ctUidSet = new Set(['$all']);
this.missingCtInExtensions = [];
this.missingCts = new Set();
this.extensionsPath = '';
}
validateModules(moduleName:keyof typeof auditConfig.moduleConfig, moduleConfig: Record<string, unknown>):keyof typeof auditConfig.moduleConfig {
if(Object.keys(moduleConfig).includes(moduleName)){
return moduleName;
}
return 'extensions'
}
cs-raj marked this conversation as resolved.
Show resolved Hide resolved

async run() {
if (!existsSync(this.folderPath)) {
Expand Down
13 changes: 10 additions & 3 deletions packages/contentstack-audit/src/modules/workflows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { join, resolve } from 'path';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { cloneDeep } from 'lodash';
import { LogFn, ConfigType, ContentTypeStruct, CtConstructorParam, ModuleConstructorParam, Workflow } from '../types';
import { ux } from '@contentstack/cli-utilities';
import { sanitizePath, ux } from '@contentstack/cli-utilities';

import auditConfig from '../config';
import { $t, auditMsg, commonMsg } from '../messages';
Expand Down Expand Up @@ -35,15 +35,22 @@ export default class Workflows {
this.fix = fix ?? false;
this.ctSchema = ctSchema;
this.workflowSchema = [];
this.moduleName = moduleName ?? 'workflows';
this.moduleName = this.validateModules(moduleName!,this.config.moduleConfig);
this.fileName = config.moduleConfig[this.moduleName].fileName;
this.folderPath = resolve(config.basePath, config.moduleConfig[this.moduleName].dirName);
this.folderPath = resolve(sanitizePath(config.basePath), sanitizePath(config.moduleConfig[this.moduleName].dirName));
this.ctUidSet = new Set(['$all']);
this.missingCtInWorkflows = [];
this.missingCts = new Set();
this.workflowPath = '';
this.isBranchFixDone = false;
}
validateModules(moduleName:keyof typeof auditConfig.moduleConfig, moduleConfig: Record<string, unknown>):keyof typeof auditConfig.moduleConfig {
if(Object.keys(moduleConfig).includes(moduleName)){
return moduleName;
}
return 'workflows'
}
cs-raj marked this conversation as resolved.
Show resolved Hide resolved

/**
* Check whether the given path for the workflow exists or not
* If path exist read
Expand Down
Loading
Loading