Skip to content

Commit

Permalink
Merge pull request #1313 from contentstack/staging
Browse files Browse the repository at this point in the history 
Sprint 52 release
  • Loading branch information
@aman19K
aman19K authored Mar 4, 2024
2 parents 0544ffc + b30b418 commit b28639a
Show file tree
Hide file tree
Showing 78 changed files with 2,073 additions and 1,035 deletions.
855 changes: 686 additions & 169 deletions package-lock.json
View file

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions packages/contentstack-audit/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ $ npm install -g @contentstack/cli-audit
$ csdx COMMAND
running command...
$ csdx (--version|-v)
@contentstack/cli-audit/1.3.5 darwin-arm64 node-v20.8.0
@contentstack/cli-audit/1.4.0 darwin-arm64 node-v20.8.0
$ csdx --help [COMMAND]
USAGE
$ csdx COMMAND
Expand Down Expand Up @@ -99,14 +99,14 @@ Perform audits and fix possible errors in the exported Contentstack data.
USAGE
$ csdx audit:fix [-c <value>] [-d <value>] [--report-path <value>] [--modules
content-types|global-fields|entries] [--copy-path <value> --copy-dir] [--fix-only
reference|global_field|json:rte|json:custom-field|blocks|group] [--columns <value> | ] [--sort <value>] [--filter
reference|global_field|json:rte|json:extension|blocks|group] [--columns <value> | ] [--sort <value>] [--filter
<value>] [--csv | --no-truncate]
FLAGS
--copy-dir Create backup from the original data.
--copy-path=<value> Provide the path to backup the copied data
--fix-only=<option>... Provide the list of fix options
<options: reference|global_field|json:rte|json:custom-field|blocks|group>
<options: reference|global_field|json:rte|json:extension|blocks|group>
--modules=<option>... Provide the list of modules to be audited
<options: content-types|global-fields|entries>
--report-path=<value> Path to store the audit reports
Expand Down Expand Up @@ -198,14 +198,14 @@ Perform audits and fix possible errors in the exported Contentstack data.
USAGE
$ csdx cm:stacks:audit:fix [-c <value>] [-d <value>] [--report-path <value>] [--modules
content-types|global-fields|entries] [--copy-path <value> --copy-dir] [--fix-only
reference|global_field|json:rte|json:custom-field|blocks|group] [--columns <value> | ] [--sort <value>] [--filter
reference|global_field|json:rte|json:extension|blocks|group] [--columns <value> | ] [--sort <value>] [--filter
<value>] [--csv | --no-truncate]
FLAGS
--copy-dir Create backup from the original data.
--copy-path=<value> Provide the path to backup the copied data
--fix-only=<option>... Provide the list of fix options
<options: reference|global_field|json:rte|json:custom-field|blocks|group>
<options: reference|global_field|json:rte|json:extension|blocks|group>
--modules=<option>... Provide the list of modules to be audited
<options: content-types|global-fields|entries>
--report-path=<value> Path to store the audit reports
Expand Down
4 changes: 2 additions & 2 deletions packages/contentstack-audit/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@contentstack/cli-audit",
"version": "1.3.5",
"version": "1.4.0",
"description": "Contentstack audit plugin",
"author": "Contentstack CLI",
"homepage": "https://github.com/contentstack/cli",
Expand All @@ -19,7 +19,7 @@
],
"dependencies": {
"@contentstack/cli-command": "~1.2.16",
"@contentstack/cli-utilities": "~1.5.11",
"@contentstack/cli-utilities": "~1.5.12",
"@oclif/plugin-help": "^5",
"@oclif/plugin-plugins": "^4.1.9",
"chalk": "^4.1.2",
Expand Down
20 changes: 19 additions & 1 deletion packages/contentstack-audit/src/config/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ const config = {
skipRefs: ['sys_assets'],
skipFieldTypes: ['taxonomy'],
modules: ['content-types', 'global-fields', 'entries'],
'fix-fields': ['reference', 'global_field', 'json:rte', 'json:custom-field', 'blocks', 'group'],
'fix-fields': ['reference', 'global_field', 'json:rte', 'json:extension', 'blocks', 'group'],
moduleConfig: {
'content-types': {
name: 'content type',
Expand All @@ -26,6 +26,24 @@ const config = {
fileName: 'locales.json',
},
},
entries: {
systemKeys: [
'uid',
'ACL',
'tags',
'locale',
'_version',
'_metadata',
'published',
'created_at',
'updated_at',
'created_by',
'updated_by',
'_in_progress',
'_restore_status',
'publish_details',
],
},
};

export default config;
146 changes: 133 additions & 13 deletions packages/contentstack-audit/src/modules/content-types.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import map from 'lodash/map';
import find from 'lodash/find';
import isEmpty from 'lodash/isEmpty';
import { join, resolve } from 'path';
import { existsSync, writeFileSync } from 'fs';
import { existsSync, readFileSync, writeFileSync } from 'fs';

import { ux } from '@contentstack/cli-utilities';

import {
Expand All @@ -19,9 +21,11 @@ import {
ReferenceFieldDataType,
ContentTypeSchemaType,
GlobalFieldSchemaTypes,
ExtensionOrAppFieldDataType,
} from '../types';
import auditConfig from '../config';
import { $t, auditFixMsg, auditMsg, commonMsg } from '../messages';
import { MarketplaceAppsInstallationData } from '../types/extension';

/* The `ContentType` class is responsible for scanning content types, looking for references, and
generating a report in JSON and CSV formats. */
Expand All @@ -33,6 +37,7 @@ export default class ContentType {
public folderPath: string;
public currentUid!: string;
public currentTitle!: string;
public extensions: string[] = [];
public inMemoryFix: boolean = false;
public gfSchema: ContentTypeStruct[];
public ctSchema: ContentTypeStruct[];
Expand Down Expand Up @@ -67,6 +72,8 @@ export default class ContentType {

this.schema = this.moduleName === 'content-types' ? this.ctSchema : this.gfSchema;

await this.prerequisiteData();

for (const schema of this.schema ?? []) {
this.currentUid = schema.uid;
this.currentTitle = schema.title;
Expand Down Expand Up @@ -96,6 +103,35 @@ export default class ContentType {
return this.missingRefs;
}

/**
* @method prerequisiteData
* The `prerequisiteData` function reads and parses JSON files to retrieve extension and marketplace
* app data, and stores them in the `extensions` array.
*/
async prerequisiteData() {
const extensionPath = resolve(this.config.basePath, 'extensions', 'extensions.json');
const marketplacePath = resolve(this.config.basePath, 'marketplace_apps', 'marketplace_apps.json');

if (existsSync(extensionPath)) {
try {
this.extensions = Object.keys(JSON.parse(readFileSync(extensionPath, 'utf8')));
} catch (error) {}
}

if (existsSync(marketplacePath)) {
try {
const marketplaceApps: MarketplaceAppsInstallationData[] = JSON.parse(readFileSync(marketplacePath, 'utf8'));

for (const app of marketplaceApps) {
const metaData = map(map(app?.ui_location?.locations, 'meta').flat(), 'extension_uid').filter(
(val) => val,
) as string[];
this.extensions.push(...metaData);
}
} catch (error) {}
}
}

/**
* The function checks if it can write the fix content to a file and if so, it writes the content as
* JSON to the specified file path.
Expand Down Expand Up @@ -157,10 +193,16 @@ export default class ContentType {
);
break;
case 'json':
if (child.field_metadata.extension) {
if (!fixTypes.includes('json:custom-field')) continue;
if ('extension' in child.field_metadata && child.field_metadata.extension) {
if (!fixTypes.includes('json:extension')) continue;
// NOTE Custom field type
} else if (child.field_metadata.allow_json_rte) {
this.missingRefs[this.currentUid].push(
...this.validateExtensionAndAppField(
[...tree, { uid: child.uid, name: child.display_name }],
child as ExtensionOrAppFieldDataType,
),
);
} else if ('allow_json_rte' in child.field_metadata && child.field_metadata.allow_json_rte) {
if (!fixTypes.includes('json:rte')) continue;
// NOTE JSON RTE field type
this.missingRefs[this.currentUid].push(
Expand Down Expand Up @@ -199,6 +241,45 @@ export default class ContentType {
return this.validateReferenceToValues(tree, field);
}

/**
* The function `validateExtensionAndAppsField` checks if a given field has a valid extension or app
* reference and returns any missing references.
* @param {Record<string, unknown>[]} tree - An array of objects representing a tree structure.
* @param {ExtensionOrAppFieldDataType} field - The `field` parameter is of type `ExtensionOrAppFieldDataType`.
* @returns The function `validateExtensionAndAppsField` returns an array of `RefErrorReturnType`
* objects.
*/
validateExtensionAndAppField(
tree: Record<string, unknown>[],
field: ExtensionOrAppFieldDataType,
): RefErrorReturnType[] {
if (this.fix) return [];

const missingRefs = [];
let { uid, extension_uid, display_name, data_type } = field;

if (!this.extensions.includes(extension_uid)) {
missingRefs.push({ uid, extension_uid, type: 'Extension or Apps' } as any);
}

return missingRefs.length
? [
{
tree,
data_type,
missingRefs,
display_name,
ct_uid: this.currentUid,
name: this.currentTitle,
treeStr: tree
.map(({ name }) => name)
.filter((val) => val)
.join(' ➜ '),
},
]
: [];
}

/**
* The function "validateGlobalField" asynchronously validates a global field by looking for a
* reference in a tree data structure.
Expand Down Expand Up @@ -298,7 +379,9 @@ export default class ContentType {

for (const reference of reference_to ?? []) {
// NOTE Can skip specific references keys (Ex, system defined keys can be skipped)
if (this.config.skipRefs.includes(reference)) continue;
if (this.config.skipRefs.includes(reference)) {
continue;
}

const refExist = find(this.ctSchema, { uid: reference });

Expand Down Expand Up @@ -350,14 +433,13 @@ export default class ContentType {
case 'json':
case 'reference':
if (data_type === 'json') {
if (field.field_metadata.extension) {
if ('extension' in field.field_metadata && field.field_metadata.extension) {
// NOTE Custom field type
if (!fixTypes.includes('json:custom-field')) return field;
if (!fixTypes.includes('json:extension')) return field;

// NOTE Fix logic

return field;
} else if (field.field_metadata.allow_json_rte) {
return this.fixMissingExtensionOrApp(tree, field as ExtensionOrAppFieldDataType);
} else if ('allow_json_rte' in field.field_metadata && field.field_metadata.allow_json_rte) {
if (!fixTypes.includes('json:rte')) return field;

return this.fixMissingReferences(tree, field as JsonRTEFieldDataType);
Expand Down Expand Up @@ -456,10 +538,10 @@ export default class ContentType {
const refErrorObj = {
tree,
display_name,
fixStatus: 'Fixed',
missingRefs: [reference_to],
ct_uid: this.currentUid,
name: this.currentTitle,
missingRefs: [reference_to],
fixStatus: this.fix ? 'Fixed' : undefined,
treeStr: tree.map(({ name }) => name).join(' ➜ '),
};

Expand Down Expand Up @@ -499,6 +581,41 @@ export default class ContentType {
.filter((val) => val) as ModularBlockType[];
}

/**
* The function checks for missing extension or app references in a given tree and fixes them if the
* fix flag is enabled.
* @param {Record<string, unknown>[]} tree - An array of objects representing a tree structure.
* @param {ExtensionOrAppFieldDataType} field - The `field` parameter is of type
* `ExtensionOrAppFieldDataType`.
* @returns If the `fix` flag is true and there are missing references (`missingRefs` is not empty),
* then `null` is returned. Otherwise, the `field` parameter is returned.
*/
fixMissingExtensionOrApp(tree: Record<string, unknown>[], field: ExtensionOrAppFieldDataType) {
const missingRefs: string[] = [];
const { uid, extension_uid, data_type, display_name } = field;

if (!this.extensions.includes(extension_uid)) {
missingRefs.push({ uid, extension_uid, type: 'Extension or Apps' } as any);
}

if (this.fix && !isEmpty(missingRefs)) {
this.missingRefs[this.currentUid].push({
tree,
data_type,
missingRefs,
display_name,
fixStatus: 'Fixed',
ct_uid: this.currentUid,
name: this.currentTitle,
treeStr: tree.map(({ name }) => name).join(' ➜ '),
});

return null
}

return field;
}

/**
* The function `fixMissingReferences` checks for missing references in a given tree and field, and
* attempts to fix them by removing the missing references from the field's `reference_to` array.
Expand All @@ -515,7 +632,9 @@ export default class ContentType {

for (const reference of reference_to ?? []) {
// NOTE Can skip specific references keys (Ex, system defined keys can be skipped)
if (this.config.skipRefs.includes(reference)) continue;
if (this.config.skipRefs.includes(reference)) {
continue;
}

const refExist = find(this.ctSchema, { uid: reference });

Expand Down Expand Up @@ -557,6 +676,7 @@ export default class ContentType {
*/
fixGroupField(tree: Record<string, unknown>[], field: GroupFieldDataType) {
const { data_type, display_name } = field;

field.schema = this.runFixOnSchema(tree, field.schema as ContentTypeSchemaType[]);

if (isEmpty(field.schema)) {
Expand Down
Loading

0 comments on commit b28639a

Please sign in to comment.