diff --git a/package-lock.json b/package-lock.json index d9e1aa3dc9..ac7438cad9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4235,11 +4235,11 @@ } }, "node_modules/axios": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.3.tgz", - "integrity": "sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==", + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.5.tgz", + "integrity": "sha512-Ii012v05KEVuUoFWmMW/UQv9aRIc3ZwkWDcM+h5Il8izZCtRVpDUfwpoFf7eOtajT3QiGR4yDUx7lPqHJULgbg==", "dependencies": { - "follow-redirects": "^1.15.0", + "follow-redirects": "^1.15.4", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -8238,9 +8238,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "node_modules/follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", + "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", "funding": [ { "type": "individual", @@ -23496,14 +23496,14 @@ "@contentstack/cli-audit": "~1.3.2", "@contentstack/cli-auth": "~1.3.17", "@contentstack/cli-cm-bootstrap": "~1.8.0", - "@contentstack/cli-cm-branches": "~1.0.19", + "@contentstack/cli-cm-branches": "~1.0.20", "@contentstack/cli-cm-bulk-publish": "~1.4.0", - "@contentstack/cli-cm-clone": "~1.8.0", + "@contentstack/cli-cm-clone": "~1.9.0", "@contentstack/cli-cm-export": "~1.10.2", "@contentstack/cli-cm-export-to-csv": "~1.6.2", - "@contentstack/cli-cm-import": "~1.12.2", + "@contentstack/cli-cm-import": "~1.13.0", "@contentstack/cli-cm-migrate-rte": "~1.4.15", - "@contentstack/cli-cm-seed": "~1.7.0", + "@contentstack/cli-cm-seed": "~1.7.1", "@contentstack/cli-command": "~1.2.17", "@contentstack/cli-config": "~1.5.1", "@contentstack/cli-launch": "~1.0.15", @@ -23923,7 +23923,7 @@ }, "packages/contentstack-branches": { "name": "@contentstack/cli-cm-branches", - "version": "1.0.19", + "version": "1.0.20", "license": "MIT", "dependencies": { "@contentstack/cli-command": "~1.2.16", @@ -24049,12 +24049,12 @@ }, "packages/contentstack-clone": { "name": "@contentstack/cli-cm-clone", - "version": "1.8.0", + "version": "1.9.0", "license": "MIT", "dependencies": { "@colors/colors": "^1.5.0", "@contentstack/cli-cm-export": "~1.10.2", - "@contentstack/cli-cm-import": "~1.12.2", + "@contentstack/cli-cm-import": "~1.13.0", "@contentstack/cli-command": "~1.2.16", "@contentstack/cli-utilities": "~1.5.10", "async": "^3.2.4", @@ -24920,14 +24920,14 @@ }, "packages/contentstack-import": { "name": "@contentstack/cli-cm-import", - "version": "1.12.2", + "version": "1.13.0", "license": "MIT", "dependencies": { + "@contentstack/cli-audit": "^1.3.2", "@contentstack/cli-command": "~1.2.16", "@contentstack/cli-utilities": "~1.5.10", "@contentstack/management": "~1.13.0", "@oclif/core": "^2.9.3", - "axios": "^1.6.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", "chalk": "^4.1.2", @@ -25378,10 +25378,10 @@ }, "packages/contentstack-seed": { "name": "@contentstack/cli-cm-seed", - "version": "1.7.0", + "version": "1.7.1", "license": "MIT", "dependencies": { - "@contentstack/cli-cm-import": "~1.12.2", + "@contentstack/cli-cm-import": "~1.13.0", "@contentstack/cli-command": "~1.2.16", "@contentstack/cli-utilities": "~1.5.10", "inquirer": "8.2.4", @@ -25398,7 +25398,7 @@ "@types/node": "^14.14.32", "@types/tar": "^6.1.3", "@types/tmp": "^0.2.0", - "axios": "^1.6.3", + "axios": "^1.6.4", "eslint": "^8.18.0", "eslint-config-oclif": "^4.0.0", "eslint-config-oclif-typescript": "^3.0.8", @@ -25462,7 +25462,7 @@ "@contentstack/management": "~1.13.0", "@contentstack/marketplace-sdk": "^1.0.1", "@oclif/core": "^2.9.3", - "axios": "^1.6.3", + "axios": "^1.6.4", "chalk": "^4.0.0", "cli-cursor": "^3.1.0", "cli-table": "^0.3.11", diff --git a/packages/contentstack-audit/src/audit-base-command.ts b/packages/contentstack-audit/src/audit-base-command.ts index 8f899224b7..aa6c15815c 100644 --- a/packages/contentstack-audit/src/audit-base-command.ts +++ b/packages/contentstack-audit/src/audit-base-command.ts @@ -36,7 +36,7 @@ export abstract class AuditBaseCommand extends BaseCommand { + async start(command: CommandNames): Promise { this.currentCommand = command; await this.promptQueue(); await this.createBackUp(); @@ -69,6 +69,8 @@ export abstract class AuditBaseCommand extends BaseCommand extends Command { this.sharedConfig = Object.assign(this.sharedConfig, { flags: this.flags }); - if (this.flags['external-config']?.config) { + if (!isEmpty(this.flags['external-config']?.config)) { this.sharedConfig = Object.assign(this.sharedConfig, this.flags['external-config']?.config); } diff --git a/packages/contentstack-audit/src/commands/cm/stacks/audit/fix.ts b/packages/contentstack-audit/src/commands/cm/stacks/audit/fix.ts index 31f22e05e5..0145b9a931 100644 --- a/packages/contentstack-audit/src/commands/cm/stacks/audit/fix.ts +++ b/packages/contentstack-audit/src/commands/cm/stacks/audit/fix.ts @@ -60,12 +60,12 @@ export default class AuditFix extends AuditBaseCommand { * The `run` function is an asynchronous function that performs an audit on different modules * (content-types, global-fields, entries) and generates a report. */ - async run(): Promise { + async run(): Promise { try { - await this.start('cm:stacks:audit:fix'); + const hasFix = await this.start('cm:stacks:audit:fix'); - if (this.flags['external-config']?.returnConfig) { - return this.sharedConfig; + if (this.flags['external-config']?.returnResponse) { + return { config: this.sharedConfig, hasFix }; } } catch (error) { this.log(error instanceof Error ? error.message : error, 'error'); diff --git a/packages/contentstack-audit/src/index.ts b/packages/contentstack-audit/src/index.ts index ff8b4c5632..b612d475f8 100644 --- a/packages/contentstack-audit/src/index.ts +++ b/packages/contentstack-audit/src/index.ts @@ -1 +1,4 @@ -export default {}; +import Audit from "./commands/cm/stacks/audit"; +import AuditFix from "./commands/cm/stacks/audit/fix"; + +export { Audit, AuditFix }; diff --git a/packages/contentstack-audit/src/modules/content-types.ts b/packages/contentstack-audit/src/modules/content-types.ts index 7f91346119..0b544992f5 100644 --- a/packages/contentstack-audit/src/modules/content-types.ts +++ b/packages/contentstack-audit/src/modules/content-types.ts @@ -102,7 +102,7 @@ export default class ContentType { let canWrite = true; if (!this.inMemoryFix && this.fix) { - if (!this.config.flags['copy-dir']) { + if (!this.config.flags['copy-dir'] && !this.config.flags['external-config']?.skipConfirm) { canWrite = this.config.flags.yes ?? (await ux.confirm(commonMsg.FIX_CONFIRMATION)); } diff --git a/packages/contentstack-audit/src/modules/entries.ts b/packages/contentstack-audit/src/modules/entries.ts index f68224cf42..586d4071b6 100644 --- a/packages/contentstack-audit/src/modules/entries.ts +++ b/packages/contentstack-audit/src/modules/entries.ts @@ -160,7 +160,7 @@ export default class Entries { let canWrite = true; if (this.fix) { - if (!this.config.flags['copy-dir']) { + if (!this.config.flags['copy-dir'] && !this.config.flags['external-config']?.skipConfirm) { canWrite = this.config.flags.yes || (await ux.confirm(commonMsg.FIX_CONFIRMATION)); } diff --git a/packages/contentstack-branches/README.md b/packages/contentstack-branches/README.md index 2cdc9f6ddf..2ecc88b069 100755 --- a/packages/contentstack-branches/README.md +++ b/packages/contentstack-branches/README.md @@ -37,7 +37,7 @@ $ npm install -g @contentstack/cli-cm-branches $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-branches/1.0.19 darwin-arm64 node-v20.8.0 +@contentstack/cli-cm-branches/1.0.20 darwin-arm64 node-v20.8.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-branches/package.json b/packages/contentstack-branches/package.json index f9fe694229..29b95f9d6b 100644 --- a/packages/contentstack-branches/package.json +++ b/packages/contentstack-branches/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/cli-cm-branches", "description": "Contentstack CLI plugin to do branches operations", - "version": "1.0.19", + "version": "1.0.20", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { diff --git a/packages/contentstack-branches/src/branch/merge-handler.ts b/packages/contentstack-branches/src/branch/merge-handler.ts index 823da0743a..aa91ede899 100644 --- a/packages/contentstack-branches/src/branch/merge-handler.ts +++ b/packages/contentstack-branches/src/branch/merge-handler.ts @@ -1,3 +1,4 @@ +import os from 'os'; import path from 'path'; import forEach from 'lodash/forEach'; import { cliux } from '@contentstack/cli-utilities'; @@ -275,7 +276,7 @@ export default class MergeHandler { deleted: [], }; - selectedMergeItems.forEach((item) => { + selectedMergeItems?.forEach((item) => { mergeContent.content_types[item.status].push(item.value); }); break; @@ -290,10 +291,16 @@ export default class MergeHandler { if (scriptFolderPath !== undefined) { cliux.success(`\nSuccess! We have generated entry migration files in the folder ${scriptFolderPath}`); cliux.print('\nWARNING!!! Migration is not intended to be run more than once. Migrated(entries/assets) will be duplicated if run more than once', {color: 'yellow'}); - - const migrationCommand = `csdx cm:stacks:migration --multiple --file-path ./${scriptFolderPath} --config {compare-branch:${mergePayload.compare_branch},file-path:./${scriptFolderPath}} --branch ${mergePayload.base_branch} --stack-api-key ${this.stackAPIKey}`; + + let migrationCommand: string; + if(os.platform() === 'win32'){ + migrationCommand = `csdx cm:stacks:migration --multiple --file-path ./${scriptFolderPath} --config compare-branch:${mergePayload.compare_branch},file-path:./${scriptFolderPath} --branch ${mergePayload.base_branch} --stack-api-key ${this.stackAPIKey}`; + }else{ + migrationCommand = `csdx cm:stacks:migration --multiple --file-path ./${scriptFolderPath} --config {compare-branch:${mergePayload.compare_branch},file-path:./${scriptFolderPath}} --branch ${mergePayload.base_branch} --stack-api-key ${this.stackAPIKey}`; + } + cliux.print( - `\nKindly follow the steps in the guide "https://www.contentstack.com/docs/developers/cli/migrate-branch-entries" to update the migration scripts, and then run the command:\n\n${migrationCommand}`, + `\nKindly follow the steps in the guide "https://www.contentstack.com/docs/developers/cli/entry-migration" to update the migration scripts, and then run the command:\n\n${migrationCommand}`, { color: 'blue' }, ); } diff --git a/packages/contentstack-branches/src/utils/create-merge-scripts.ts b/packages/contentstack-branches/src/utils/create-merge-scripts.ts index 239cb86a27..95c2cf1bbc 100644 --- a/packages/contentstack-branches/src/utils/create-merge-scripts.ts +++ b/packages/contentstack-branches/src/utils/create-merge-scripts.ts @@ -30,7 +30,6 @@ export function generateMergeScripts(mergeSummary, mergeJobUID) { merge_existing_new: entryCreateUpdateScript, merge_existing: entryUpdateScript, merge_new: entryCreateScript, - ignore: entryCreateUpdateScript, }; const processContentTypes = (contentTypes, messageType) => { @@ -66,6 +65,8 @@ export function getContentTypeMergeStatus(status) { return 'created'; } else if (status === 'merge_existing_new') { return 'created_updated'; + } else if (status === 'ignore') { + return; } else { return ''; } diff --git a/packages/contentstack-branches/src/utils/entry-create-script.ts b/packages/contentstack-branches/src/utils/entry-create-script.ts index e40f18cb16..228202ae59 100644 --- a/packages/contentstack-branches/src/utils/entry-create-script.ts +++ b/packages/contentstack-branches/src/utils/entry-create-script.ts @@ -2,6 +2,14 @@ export function entryCreateScript(contentType) { return ` const fs = require('fs'); const path = require('path'); + const { marked } = require('marked'); + const has = require('lodash/has'); + const isArray = require('lodash/isArray'); + const isObject = require('lodash/isObject'); + const omit = require('lodash/omit'); + const compact = require('lodash/compact') + const isPlainObject = require('lodash/isPlainObject'); + const {cliux, LoggerService} = require('@contentstack/cli-utilities') module.exports = async ({ migration, stackSDKInstance, managementAPIClient, config, branch, apiKey }) => { const keysToRemove = [ 'content_type_uid', @@ -31,20 +39,18 @@ export function entryCreateScript(contentType) { let assetUIDMapper = {}; let assetUrlMapper = {}; let assetRefPath = {}; - let isAssetDownload = false; + let downloadedAssets = []; + let parent=[]; + let logger; function getValueByPath(obj, path) { return path.split('[').reduce((o, key) => o && o[key.replace(/\]$/, '')], obj); } - function updateValueByPath(obj, path, newValue, type, fileIndex) { + function updateValueByPath(obj, path, newValue) { path.split('[').reduce((o, key, index, arr) => { if (index === arr.length - 1) { - if (type === 'file') { - o[key.replace(/]$/, '')][fileIndex] = newValue; - } else { o[key.replace(/]$/, '')][0].uid = newValue; - } } else { return o[key.replace(/\]$/, '')]; } @@ -77,35 +83,52 @@ export function entryCreateScript(contentType) { return references; }; - const findAssets = function (schema, entry, refPath, path) { + const findAssets = function (schema, entry) { for (const i in schema) { - const currentPath = path ? path + '[' + schema[i].uid : schema[i].uid; if (schema[i].data_type === 'group' || schema[i].data_type === 'global_field') { - findAssets(schema[i].schema, entry, refPath, currentPath + '[0]'); - } else if (schema[i].data_type === 'blocks') { - for (const block in schema[i].blocks) { + parent.push(schema[i].uid); + findAssets(schema[i].schema, entry); + parent.pop(); + } + if (schema[i].data_type === 'blocks') { + for (const j = 0; j < schema[i].blocks; j++) { { - if (schema[i].blocks[block].schema) { - findAssets( - schema[i].blocks[block].schema, - entry, - refPath, - currentPath + '[' + block + '][' + schema[i].blocks[block].uid + ']', - ); + if (schema[i].blocks[j].schema) { + parent.push(schema[i].uid); + parent.push(j); + parent.push(schema[i].blocks[j].uid); + findAssets(schema[i].blocks[j].schema, entry); + parent.pop(); + parent.pop(); + parent.pop(); } } } - } else if (schema[i].data_type === 'json' && schema[i].field_metadata.rich_text_type) { - findAssetIdsFromJsonRte(entry, schema, refPath, path); - } else if ( + } + if (schema[i].data_type === 'json' && schema[i].field_metadata.rich_text_type) { + parent.push(schema[i].uid); + findAssetIdsFromJsonRte(entry, schema); + parent.pop(); + } + if ( schema[i].data_type === 'text' && schema[i].field_metadata && (schema[i].field_metadata.markdown || schema[i].field_metadata.rich_text_type) ) { + parent.push(schema[i].uid); findFileUrls(schema[i], entry); - } else if (schema[i].data_type === 'file') { - refPath.push(currentPath) - const imgDetails = getValueByPath(entry, currentPath); + if (schema[i].field_metadata.rich_text_type) { + findAssetIdsFromHtmlRte(entry, schema[i]); + } + parent.pop(); + } + if (schema[i].data_type === 'file') { + parent.push(schema[i].uid); + let updatedEntry = entry; + for (let i = 0; i < parent.length; i++) { + updatedEntry = updatedEntry[parent[i]]; + } + const imgDetails = updatedEntry; if (schema[i].multiple) { if (imgDetails && imgDetails.length) { imgDetails.forEach((img) => { @@ -133,10 +156,21 @@ export function entryCreateScript(contentType) { cAssetDetails.push(obj); } } + parent.pop(); } } }; + function findAssetIdsFromHtmlRte(entryObj, ctSchema) { + const regex = / { - let imgDetails = entry[refPath]; - if (imgDetails !== undefined) { - if (imgDetails && !Array.isArray(imgDetails)) { - entry[refPath] = assetUIDMapper[imgDetails.uid]; - } else if (imgDetails && Array.isArray(imgDetails)) { - for (let i = 0; i < imgDetails.length; i++) { - const img = imgDetails[i]; - entry[refPath][i] = assetUIDMapper[img.uid]; - } - } - } else { - imgDetails = getValueByPath(entry, refPath); - if (imgDetails && !Array.isArray(imgDetails)) { - const imgUID = imgDetails?.uid; - updateValueByPath(entry, refPath, assetUIDMapper[imgUID], 'file', 0); - } else if (imgDetails && Array.isArray(imgDetails)) { - for (let i = 0; i < imgDetails.length; i++) { - const img = imgDetails[i]; - const imgUID = img?.uid; - updateValueByPath(entry, refPath, assetUIDMapper[imgUID], 'file', i); - } - } - } - }); + let updatedEntry = Object.assign({},entry); + entry = updateFileFields(updatedEntry, entry, null) entry = JSON.stringify(entry); const assetUrls = cAssetDetails.map((asset) => asset.url); const assetUIDs = cAssetDetails.map((asset) => asset.uid); @@ -299,6 +310,65 @@ export function entryCreateScript(contentType) { }); return JSON.parse(entry); }; + + function updateFileFields( + object, + parent, + pos + ) { + if (isPlainObject(object) && has(object, 'filename') && has(object, 'uid')) { + if (typeof pos !== 'undefined') { + if (typeof pos === 'number' || typeof pos === 'string') { + const replacer = () => { + if (assetUIDMapper.hasOwnProperty(object.uid)) { + parent[pos] = assetUIDMapper[object.uid]; + } else { + parent[pos] = ''; + } + }; + + if (parent.uid && assetUIDMapper[parent.uid]) { + parent.uid = assetUIDMapper[parent.uid]; + } + + if ( + object && + isObject(parent[pos]) && + parent[pos].uid && + parent[pos].url && + has(parent, 'asset') && + has(parent, '_content_type_uid') && + parent._content_type_uid === 'sys_assets' + ) { + if ( + has(parent, 'asset') && + has(parent, '_content_type_uid') && + parent._content_type_uid === 'sys_assets' + ) { + parent = omit(parent, ['asset']); + } + + if (object.uid && assetUIDMapper[object.uid]) { + object.uid = assetUIDMapper[object.uid]; + } + if (object.url && assetUrlMapper[object.url]) { + object.url = assetUrlMapper[object.url]; + } + } else { + replacer(); + } + } + } + } else if (isPlainObject(object)) { + for (let key in object) updateFileFields(object[key], object, key); + } else if (isArray(object) && object.length) { + for (let i = 0; i <= object.length; i++){ + updateFileFields(object[i], object, i); + } + parent[pos] = compact(object); + } + return object; + } const checkAndDownloadAsset = async function (cAsset) { const assetUID = cAsset?.uid; @@ -315,7 +385,6 @@ export function entryCreateScript(contentType) { return false; } else { - isAssetDownload = true; const cAssetDetail = await managementAPIClient .stack({ api_key: stackSDKInstance.api_key, branch_uid: compareBranch }) .asset(assetUID) @@ -357,8 +426,8 @@ export function entryCreateScript(contentType) { const uploadAssets = async function () { const assetFolderMap = JSON.parse(fs.readFileSync(path.resolve(filePath, 'folder-mapper.json'), 'utf8')); const stackAPIClient = managementAPIClient.stack({ api_key: stackSDKInstance.api_key, branch_uid: branch }); - for (let i = 0; i < cAssetDetails?.length; i++) { - const asset = cAssetDetails[i]; + for (let i = 0; i < downloadedAssets?.length; i++) { + const asset = downloadedAssets[i]; let requestOption = {}; requestOption.parent_uid = assetFolderMap[asset.parent_uid] || asset.parent_uid; @@ -401,6 +470,11 @@ export function entryCreateScript(contentType) { successTitle: 'Entries Created Successfully', failedTitle: 'Failed to create entries', task: async () => { + //logger file + if(!fs.existsSync(path.join(filePath, 'entry-migration'))){ + logger = new LoggerService(filePath, 'entry-migration'); + } + const compareBranchEntries = await managementAPIClient .stack({ api_key: stackSDKInstance.api_key, branch_uid: compareBranch }) .contentType('${contentType}') @@ -432,9 +506,10 @@ export function entryCreateScript(contentType) { const updatedCAsset = await checkAndDownloadAsset(asset); if (updatedCAsset) { cAssetDetails[i] = updatedCAsset; + downloadedAssets.push(updatedCAsset) } } - if (isAssetDownload) await uploadAssets(); + if (downloadedAssets?.length) await uploadAssets(); } let flag = { @@ -445,7 +520,17 @@ export function entryCreateScript(contentType) { async function updateEntry(entry, entryDetails) { Object.assign(entry, { ...entryDetails }); - await entry.update(); + await entry.update().catch(err => { + let errorMsg = 'Entry update failed for uid: ' + entry?.uid + ', title: ' + entry?.title + '. '; + if(err?.errors?.title){ + errorMsg += 'title'+ err?.errors?.title; + }else if(err?.errors?.entry){ + errorMsg += err?.errors?.entry; + }else{ + errorMsg += (err?.entry?.errorMessage || err?.errorMessage || err?.message) ?? 'Something went wrong!'; + } + logger.error(errorMsg) + }); } async function updateReferences(entryDetails, baseEntry, references) { @@ -477,8 +562,16 @@ export function entryCreateScript(contentType) { compareFilteredProperties.forEach(async (entryDetails) => { if(entryDetails !== undefined){ entryDetails = updateAssetDetailsInEntries(entryDetails); - let createdEntry = await stackSDKInstance.contentType('${contentType}').entry().create({ entry: entryDetails }).catch(error => { - throw error; + let createdEntry = await stackSDKInstance.contentType('${contentType}').entry().create({ entry: entryDetails }).catch(err => { + let errorMsg = 'Entry creation failed for contentType: ' + contentType + ', title: ' + entryDetails?.title + '. '; + if(err?.errors?.title){ + errorMsg += err?.errors?.title; + }else if(err?.errors?.entry){ + errorMsg += err?.errors?.entry; + }else{ + errorMsg += (err?.entry?.errorMessage || err?.errorMessage || err?.message) ?? 'Something went wrong!'; + } + logger.error(errorMsg) }); if(createdEntry){ if (flag.references) { diff --git a/packages/contentstack-branches/src/utils/entry-create-update-script.ts b/packages/contentstack-branches/src/utils/entry-create-update-script.ts index 194a218aad..f4e8b8048c 100644 --- a/packages/contentstack-branches/src/utils/entry-create-update-script.ts +++ b/packages/contentstack-branches/src/utils/entry-create-update-script.ts @@ -2,6 +2,14 @@ export function entryCreateUpdateScript(contentType) { return ` const fs = require('fs'); const path = require('path'); + const { marked } = require('marked'); + const has = require('lodash/has'); + const isArray = require('lodash/isArray'); + const isObject = require('lodash/isObject'); + const omit = require('lodash/omit'); + const compact = require('lodash/compact') + const isPlainObject = require('lodash/isPlainObject'); + const {cliux, LoggerService} = require('@contentstack/cli-utilities') module.exports = async ({ migration, stackSDKInstance, managementAPIClient, config, branch, apiKey }) => { const keysToRemove = [ 'content_type_uid', @@ -29,10 +37,12 @@ export function entryCreateUpdateScript(contentType) { let assetDirPath = path.resolve(filePath, 'assets'); let assetDetails = []; let newAssetDetails = []; + let downloadedAssets = []; let assetUIDMapper = {}; let assetUrlMapper = {}; let assetRefPath = {}; - let isAssetDownload = false; + let parent=[]; + let logger; function converter(data) { let arr = []; @@ -59,14 +69,10 @@ export function entryCreateUpdateScript(contentType) { return path.split('[').reduce((o, key) => o && o[key.replace(/\]$/, '')], obj); } - function updateValueByPath(obj, path, newValue, type, fileIndex) { + function updateValueByPath(obj, path, newValue) { path.split('[').reduce((o, key, index, arr) => { if (index === arr.length - 1) { - if (type === 'file') { - o[key.replace(/]$/, '')][fileIndex] = newValue; - } else { o[key.replace(/]$/, '')][0].uid = newValue; - } } else { return o[key.replace(/\]$/, '')]; } @@ -99,65 +105,93 @@ export function entryCreateUpdateScript(contentType) { return references; }; - const findAssets = function (schema, entry, refPath, path) { - for (const i in schema) { - const currentPath = path ? path + '[' + schema[i].uid : schema[i].uid; - if (schema[i].data_type === 'group' || schema[i].data_type === 'global_field') { - findAssets(schema[i].schema, entry, refPath, currentPath + '[0]'); - } else if (schema[i].data_type === 'blocks') { - for (const block in schema[i].blocks) { - { - if (schema[i].blocks[block].schema) { - findAssets( - schema[i].blocks[block].schema, - entry, - refPath, - currentPath + '[' + block + '][' + schema[i].blocks[block].uid + ']', - ); + const findAssets = function (schema, entry) { + for (const i in schema) { + if (schema[i].data_type === 'group' || schema[i].data_type === 'global_field') { + parent.push(schema[i].uid); + findAssets(schema[i].schema, entry); + parent.pop(); + } + if (schema[i].data_type === 'blocks') { + for (const j = 0; j < schema[i].blocks; j++) { + { + if (schema[i].blocks[j].schema) { + parent.push(schema[i].uid); + parent.push(j); + parent.push(schema[i].blocks[j].uid); + findAssets(schema[i].blocks[j].schema, entry); + parent.pop(); + parent.pop(); + parent.pop(); + } } } } - } else if (schema[i].data_type === 'json' && schema[i].field_metadata.rich_text_type) { - findAssetIdsFromJsonRte(entry, schema, refPath, path); - } else if ( - schema[i].data_type === 'text' && - schema[i].field_metadata && - (schema[i].field_metadata.markdown || schema[i].field_metadata.rich_text_type) - ) { - findFileUrls(schema[i], entry); - } else if (schema[i].data_type === 'file') { - refPath.push(currentPath) - const imgDetails = getValueByPath(entry, currentPath); - if (schema[i].multiple) { - if (imgDetails && imgDetails.length) { - imgDetails.forEach((img) => { + if (schema[i].data_type === 'json' && schema[i].field_metadata.rich_text_type) { + parent.push(schema[i].uid); + findAssetIdsFromJsonRte(entry, schema); + parent.pop(); + } + if ( + schema[i].data_type === 'text' && + schema[i].field_metadata && + (schema[i].field_metadata.markdown || schema[i].field_metadata.rich_text_type) + ) { + parent.push(schema[i].uid); + findFileUrls(schema[i], entry); + if (schema[i].field_metadata.rich_text_type) { + findAssetIdsFromHtmlRte(entry, schema[i]); + } + parent.pop(); + } + if (schema[i].data_type === 'file') { + parent.push(schema[i].uid); + let updatedEntry = entry; + for (let i = 0; i < parent.length; i++) { + updatedEntry = updatedEntry[parent[i]]; + } + const imgDetails = updatedEntry; + if (schema[i].multiple) { + if (imgDetails && imgDetails.length) { + imgDetails.forEach((img) => { + const obj = { + uid: img.uid, + parent_uid: img.parent_uid, + description: img.description, + title: img.title, + filename: img.filename, + url: img.url, + }; + assetDetails.push(obj); + }); + } + } else { + if (imgDetails) { const obj = { - uid: img.uid, - parent_uid: img.parent_uid, - description: img.description, - title: img.title, - filename: img.filename, - url: img.url, + uid: imgDetails.uid, + parent_uid: imgDetails.parent_uid, + description: imgDetails.description, + title: imgDetails.title, + filename: imgDetails.filename, + url: imgDetails.url, }; assetDetails.push(obj); - }); - } - } else { - if (imgDetails) { - const obj = { - uid: imgDetails.uid, - parent_uid: imgDetails.parent_uid, - description: imgDetails.description, - title: imgDetails.title, - filename: imgDetails.filename, - url: imgDetails.url, - }; - assetDetails.push(obj); + } } + parent.pop(); } } + }; + + function findAssetIdsFromHtmlRte(entryObj, ctSchema) { + const regex = / { - let imgDetails = entry[refPath]; - if (imgDetails !== undefined) { - if (imgDetails && !Array.isArray(imgDetails)) { - entry[refPath] = assetUIDMapper[imgDetails.uid]; - } else if (imgDetails && Array.isArray(imgDetails)) { - for (let i = 0; i < imgDetails.length; i++) { - const img = imgDetails[i]; - entry[refPath][i] = assetUIDMapper[img.uid]; - } - } - } else { - imgDetails = getValueByPath(entry, refPath); - if (imgDetails && !Array.isArray(imgDetails)) { - const imgUID = imgDetails?.uid; - updateValueByPath(entry, refPath, assetUIDMapper[imgUID], 'file', 0); - } else if (imgDetails && Array.isArray(imgDetails)) { - for (let i = 0; i < imgDetails.length; i++) { - const img = imgDetails[i]; - const imgUID = img?.uid; - updateValueByPath(entry, refPath, assetUIDMapper[imgUID], 'file', i); - } - } - } - }); + let updatedEntry = Object.assign({},entry); + entry = updateFileFields(updatedEntry, entry, null) entry = JSON.stringify(entry); const assetUrls = assetDetails.map((asset) => asset.url); const assetUIDs = assetDetails.map((asset) => asset.uid); @@ -321,6 +332,65 @@ export function entryCreateUpdateScript(contentType) { }); return JSON.parse(entry); }; + + function updateFileFields( + object, + parent, + pos + ) { + if (isPlainObject(object) && has(object, 'filename') && has(object, 'uid')) { + if (typeof pos !== 'undefined') { + if (typeof pos === 'number' || typeof pos === 'string') { + const replacer = () => { + if (assetUIDMapper.hasOwnProperty(object.uid)) { + parent[pos] = assetUIDMapper[object.uid]; + } else { + parent[pos] = ''; + } + }; + + if (parent.uid && assetUIDMapper[parent.uid]) { + parent.uid = assetUIDMapper[parent.uid]; + } + + if ( + object && + isObject(parent[pos]) && + parent[pos].uid && + parent[pos].url && + has(parent, 'asset') && + has(parent, '_content_type_uid') && + parent._content_type_uid === 'sys_assets' + ) { + if ( + has(parent, 'asset') && + has(parent, '_content_type_uid') && + parent._content_type_uid === 'sys_assets' + ) { + parent = omit(parent, ['asset']); + } + + if (object.uid && assetUIDMapper[object.uid]) { + object.uid = assetUIDMapper[object.uid]; + } + if (object.url && assetUrlMapper[object.url]) { + object.url = assetUrlMapper[object.url]; + } + } else { + replacer(); + } + } + } + } else if (isPlainObject(object)) { + for (let key in object) updateFileFields(object[key], object, key); + } else if (isArray(object) && object.length) { + for (let i = 0; i <= object.length; i++){ + updateFileFields(object[i], object, i); + } + parent[pos] = compact(object); + } + return object; + } const checkAndDownloadAsset = async function (cAsset) { if (cAsset) { @@ -337,7 +407,6 @@ export function entryCreateUpdateScript(contentType) { return false; } else { - isAssetDownload = true; const cAssetDetail = await managementAPIClient .stack({ api_key: stackSDKInstance.api_key, branch_uid: compareBranch }) .asset(assetUID) @@ -379,8 +448,8 @@ export function entryCreateUpdateScript(contentType) { const uploadAssets = async function () { const assetFolderMap = JSON.parse(fs.readFileSync(path.resolve(filePath, 'folder-mapper.json'), 'utf8')); const stackAPIClient = managementAPIClient.stack({ api_key: stackSDKInstance.api_key, branch_uid: branch }); - for (let i = 0; i < assetDetails?.length; i++) { - const asset = assetDetails[i]; + for (let i = 0; i < downloadedAssets?.length; i++) { + const asset = downloadedAssets[i]; let requestOption = {}; requestOption.parent_uid = assetFolderMap[asset.parent_uid] || asset.parent_uid; @@ -416,6 +485,10 @@ export function entryCreateUpdateScript(contentType) { successMessage: 'Entries Updated Successfully', failedMessage: 'Failed to update entries', task: async () => { + //logger file + if(!fs.existsSync(path.join(filePath, 'entry-migration'))){ + logger = new LoggerService(filePath, 'entry-migration'); + } let compareBranchEntries = await managementAPIClient .stack({ api_key: stackSDKInstance.api_key, branch_uid: compareBranch }) .contentType('${contentType}') @@ -451,9 +524,10 @@ export function entryCreateUpdateScript(contentType) { const updatedCAsset = await checkAndDownloadAsset(asset); if(updatedCAsset){ newAssetDetails[i] = updatedCAsset; + downloadedAssets.push(updatedCAsset) } } - if (isAssetDownload) await uploadAssets(); + if (downloadedAssets?.length) await uploadAssets(); } let flag = { @@ -465,7 +539,17 @@ export function entryCreateUpdateScript(contentType) { async function updateEntry(entry, entryDetails) { if (entry) { Object.assign(entry, { ...entryDetails }); - await entry.update(); + await entry.update().catch(err => { + let errorMsg = 'Entry update failed for uid: ' + entry?.uid + ', title: ' + entry?.title + '.'; + if(err?.errors?.title){ + errorMsg += err?.errors?.title; + }else if(err?.errors?.entry){ + errorMsg += err?.errors?.entry; + }else{ + errorMsg += (err?.entry?.errorMessage || err?.errorMessage || err?.message) ?? 'Something went wrong!'; + } + logger.error(errorMsg) + }); } } diff --git a/packages/contentstack-branches/src/utils/entry-update-script.ts b/packages/contentstack-branches/src/utils/entry-update-script.ts index 19b6549bda..ce6257a4d1 100644 --- a/packages/contentstack-branches/src/utils/entry-update-script.ts +++ b/packages/contentstack-branches/src/utils/entry-update-script.ts @@ -2,6 +2,14 @@ export function entryUpdateScript(contentType) { return ` const fs = require('fs'); const path = require('path'); + const { marked } = require('marked'); + const has = require('lodash/has'); + const isArray = require('lodash/isArray'); + const isObject = require('lodash/isObject'); + const omit = require('lodash/omit'); + const compact = require('lodash/compact') + const isPlainObject = require('lodash/isPlainObject'); + const {cliux, LoggerService} = require('@contentstack/cli-utilities') module.exports = async ({ migration, stackSDKInstance, managementAPIClient, config, branch, apiKey }) => { const keysToRemove = [ 'content_type_uid', @@ -29,10 +37,12 @@ export function entryUpdateScript(contentType) { let assetDirPath = path.resolve(filePath, 'assets'); let assetDetails = []; let newAssetDetails = []; + let downloadedAssets = []; let assetUIDMapper = {}; let assetUrlMapper = {}; let assetRefPath = {}; - let isAssetDownload = false; + let parent=[]; + let logger; function converter(data) { let arr = []; @@ -59,14 +69,10 @@ export function entryUpdateScript(contentType) { return path.split('[').reduce((o, key) => o && o[key.replace(/\]$/, '')], obj); } - function updateValueByPath(obj, path, newValue, type, fileIndex) { + function updateValueByPath(obj, path, newValue) { path.split('[').reduce((o, key, index, arr) => { if (index === arr.length - 1) { - if (type === 'file') { - o[key.replace(/]$/, '')][fileIndex] = newValue; - } else { o[key.replace(/]$/, '')][0].uid = newValue; - } } else { return o[key.replace(/\]$/, '')]; } @@ -99,35 +105,52 @@ export function entryUpdateScript(contentType) { return references; }; - const findAssets = function (schema, entry, refPath, path) { + const findAssets = function (schema, entry) { for (const i in schema) { - const currentPath = path ? path + '[' + schema[i].uid : schema[i].uid; if (schema[i].data_type === 'group' || schema[i].data_type === 'global_field') { - findAssets(schema[i].schema, entry, refPath, currentPath + '[0]'); - } else if (schema[i].data_type === 'blocks') { - for (const block in schema[i].blocks) { + parent.push(schema[i].uid); + findAssets(schema[i].schema, entry); + parent.pop(); + } + if (schema[i].data_type === 'blocks') { + for (const j = 0; j < schema[i].blocks; j++) { { - if (schema[i].blocks[block].schema) { - findAssets( - schema[i].blocks[block].schema, - entry, - refPath, - currentPath + '[' + block + '][' + schema[i].blocks[block].uid + ']', - ); + if (schema[i].blocks[j].schema) { + parent.push(schema[i].uid); + parent.push(j); + parent.push(schema[i].blocks[j].uid); + findAssets(schema[i].blocks[j].schema, entry); + parent.pop(); + parent.pop(); + parent.pop(); } } } - } else if (schema[i].data_type === 'json' && schema[i].field_metadata.rich_text_type) { - findAssetIdsFromJsonRte(entry, schema, refPath, path); - } else if ( + } + if (schema[i].data_type === 'json' && schema[i].field_metadata.rich_text_type) { + parent.push(schema[i].uid); + findAssetIdsFromJsonRte(entry, schema); + parent.pop(); + } + if ( schema[i].data_type === 'text' && schema[i].field_metadata && (schema[i].field_metadata.markdown || schema[i].field_metadata.rich_text_type) ) { + parent.push(schema[i].uid); findFileUrls(schema[i], entry); - } else if (schema[i].data_type === 'file') { - refPath.push(currentPath) - const imgDetails = getValueByPath(entry, currentPath); + if (schema[i].field_metadata.rich_text_type) { + findAssetIdsFromHtmlRte(entry, schema[i]); + } + parent.pop(); + } + if (schema[i].data_type === 'file') { + parent.push(schema[i].uid); + let updatedEntry = entry; + for (let i = 0; i < parent.length; i++) { + updatedEntry = updatedEntry[parent[i]]; + } + const imgDetails = updatedEntry; if (schema[i].multiple) { if (imgDetails && imgDetails.length) { imgDetails.forEach((img) => { @@ -155,10 +178,21 @@ export function entryUpdateScript(contentType) { assetDetails.push(obj); } } + parent.pop(); } } }; + function findAssetIdsFromHtmlRte(entryObj, ctSchema) { + const regex = / { - let imgDetails = entry[refPath]; - if (imgDetails !== undefined) { - if (imgDetails && !Array.isArray(imgDetails)) { - entry[refPath] = assetUIDMapper[imgDetails.uid]; - } else if (imgDetails && Array.isArray(imgDetails)) { - for (let i = 0; i < imgDetails.length; i++) { - const img = imgDetails[i]; - entry[refPath][i] = assetUIDMapper[img.uid]; - } - } - } else { - imgDetails = getValueByPath(entry, refPath); - if (imgDetails && !Array.isArray(imgDetails)) { - const imgUID = imgDetails?.uid; - updateValueByPath(entry, refPath, assetUIDMapper[imgUID], 'file', 0); - } else if (imgDetails && Array.isArray(imgDetails)) { - for (let i = 0; i < imgDetails.length; i++) { - const img = imgDetails[i]; - const imgUID = img?.uid; - updateValueByPath(entry, refPath, assetUIDMapper[imgUID], 'file', i); - } - } - } - }); + let updatedEntry = Object.assign({},entry); + entry = updateFileFields(updatedEntry, entry, null) entry = JSON.stringify(entry); const assetUrls = assetDetails.map((asset) => asset.url); const assetUIDs = assetDetails.map((asset) => asset.uid); @@ -321,6 +332,65 @@ export function entryUpdateScript(contentType) { }); return JSON.parse(entry); }; + + function updateFileFields( + object, + parent, + pos + ) { + if (isPlainObject(object) && has(object, 'filename') && has(object, 'uid')) { + if (typeof pos !== 'undefined') { + if (typeof pos === 'number' || typeof pos === 'string') { + const replacer = () => { + if (assetUIDMapper.hasOwnProperty(object.uid)) { + parent[pos] = assetUIDMapper[object.uid]; + } else { + parent[pos] = ''; + } + }; + + if (parent.uid && assetUIDMapper[parent.uid]) { + parent.uid = assetUIDMapper[parent.uid]; + } + + if ( + object && + isObject(parent[pos]) && + parent[pos].uid && + parent[pos].url && + has(parent, 'asset') && + has(parent, '_content_type_uid') && + parent._content_type_uid === 'sys_assets' + ) { + if ( + has(parent, 'asset') && + has(parent, '_content_type_uid') && + parent._content_type_uid === 'sys_assets' + ) { + parent = omit(parent, ['asset']); + } + + if (object.uid && assetUIDMapper[object.uid]) { + object.uid = assetUIDMapper[object.uid]; + } + if (object.url && assetUrlMapper[object.url]) { + object.url = assetUrlMapper[object.url]; + } + } else { + replacer(); + } + } + } + } else if (isPlainObject(object)) { + for (let key in object) updateFileFields(object[key], object, key); + } else if (isArray(object) && object.length) { + for (let i = 0; i <= object.length; i++){ + updateFileFields(object[i], object, i); + } + parent[pos] = compact(object); + } + return object; + } const checkAndDownloadAsset = async function (cAsset) { if (cAsset) { @@ -337,7 +407,6 @@ export function entryUpdateScript(contentType) { return false; } else { - isAssetDownload = true; const cAssetDetail = await managementAPIClient .stack({ api_key: stackSDKInstance.api_key, branch_uid: compareBranch }) .asset(assetUID) @@ -379,8 +448,8 @@ export function entryUpdateScript(contentType) { const uploadAssets = async function () { const assetFolderMap = JSON.parse(fs.readFileSync(path.resolve(filePath, 'folder-mapper.json'), 'utf8')); const stackAPIClient = managementAPIClient.stack({ api_key: stackSDKInstance.api_key, branch_uid: branch }); - for (let i = 0; i < assetDetails?.length; i++) { - const asset = assetDetails[i]; + for (let i = 0; i < downloadedAssets?.length; i++) { + const asset = downloadedAssets[i]; let requestOption = {}; requestOption.parent_uid = assetFolderMap[asset.parent_uid] || asset.parent_uid; @@ -415,6 +484,11 @@ export function entryUpdateScript(contentType) { successMessage: 'Entries Updated Successfully', failedMessage: 'Failed to update entries', task: async () => { + //logger file + if(!fs.existsSync(path.join(filePath, 'entry-migration'))){ + logger = new LoggerService(filePath, 'entry-migration'); + } + let compareBranchEntries = await managementAPIClient .stack({ api_key: stackSDKInstance.api_key, branch_uid: compareBranch }) .contentType('${contentType}') @@ -450,9 +524,10 @@ export function entryUpdateScript(contentType) { const updatedCAsset = await checkAndDownloadAsset(asset); if(updatedCAsset){ newAssetDetails[i] = updatedCAsset; + downloadedAssets.push(updatedCAsset); } } - if (isAssetDownload) await uploadAssets(); + if (downloadedAssets?.length) await uploadAssets(); } let flag = { @@ -463,7 +538,17 @@ export function entryUpdateScript(contentType) { async function updateEntry(entry, entryDetails) { Object.assign(entry, { ...entryDetails }); - await entry.update(); + await entry.update().catch(err => { + let errorMsg = 'Entry update failed for uid: ' + entry?.uid + ', title: ' + entry?.title + '.'; + if(err?.errors?.title){ + errorMsg += err?.errors?.title; + }else if(err?.errors?.entry){ + errorMsg += err?.errors?.entry; + }else{ + errorMsg += (err?.entry?.errorMessage || err?.errorMessage || err?.message) ?? 'Something went wrong!'; + } + logger.error(errorMsg) + }); } async function updateReferences(entryDetails, baseEntry, references) { diff --git a/packages/contentstack-clone/package.json b/packages/contentstack-clone/package.json index e0d7614279..46e1791133 100644 --- a/packages/contentstack-clone/package.json +++ b/packages/contentstack-clone/package.json @@ -1,12 +1,12 @@ { "name": "@contentstack/cli-cm-clone", "description": "Contentstack stack clone plugin", - "version": "1.8.0", + "version": "1.9.0", "author": "Contentstack", "bugs": "https://github.com/rohitmishra209/cli-cm-clone/issues", "dependencies": { "@contentstack/cli-cm-export": "~1.10.2", - "@contentstack/cli-cm-import": "~1.12.2", + "@contentstack/cli-cm-import": "~1.13.0", "@contentstack/cli-command": "~1.2.16", "@contentstack/cli-utilities": "~1.5.10", "@colors/colors": "^1.5.0", diff --git a/packages/contentstack-clone/src/commands/cm/stacks/clone.js b/packages/contentstack-clone/src/commands/cm/stacks/clone.js index 12542e2cf8..6f0e1d8766 100644 --- a/packages/contentstack-clone/src/commands/cm/stacks/clone.js +++ b/packages/contentstack-clone/src/commands/cm/stacks/clone.js @@ -29,6 +29,7 @@ class StackCloneCommand extends Command { const listOfTokens = configHandler.get('tokens'); config.forceStopMarketplaceAppsPrompt = yes; + config.skipAudit = cloneCommandFlags['skip-audit']; if (cloneType) { config.cloneType = cloneType; @@ -245,6 +246,9 @@ b) Structure with content (all modules including entries & assets) required: false, description: '[Optional] Override marketplace prompts', }), + 'skip-audit': flags.boolean({ + description: 'Skips the audit fix.', + }), }; StackCloneCommand.usage = diff --git a/packages/contentstack-clone/src/lib/util/clone-handler.js b/packages/contentstack-clone/src/lib/util/clone-handler.js index c594827c46..e471e29b26 100644 --- a/packages/contentstack-clone/src/lib/util/clone-handler.js +++ b/packages/contentstack-clone/src/lib/util/clone-handler.js @@ -655,6 +655,8 @@ class CloneHandler { cmd.push('--import-webhook-status', config.importWebhookStatus); } + if (config.skipAudit) cmd.push('--skip-audit'); + if (config.forceStopMarketplaceAppsPrompt) cmd.push('-y'); fs.writeFileSync(path.join(__dirname, 'dummyConfig.json'), JSON.stringify(config)); diff --git a/packages/contentstack-import/README.md b/packages/contentstack-import/README.md index 7e2ac01355..12b43d4a34 100644 --- a/packages/contentstack-import/README.md +++ b/packages/contentstack-import/README.md @@ -47,7 +47,7 @@ $ npm install -g @contentstack/cli-cm-import $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-import/1.12.2 darwin-arm64 node-v20.8.0 +@contentstack/cli-cm-import/1.12.2 darwin-arm64 node-v20.10.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND @@ -83,6 +83,7 @@ FLAGS --replace-existing Replaces the existing module in the target stack. --skip-app-recreation [optional] Skip private apps recreation if already exist + --skip-audit Skips the audit fix. --skip-existing Skips the module exists warning messages. DESCRIPTION @@ -106,7 +107,7 @@ EXAMPLES $ csdx cm:stacks:import --alias --config - $ csdx cm:stacks:import --branch --yes + $ csdx cm:stacks:import --branch --yes --skip-audit ``` ## `csdx cm:stacks:import [-c ] [-k ] [-d ] [-a ] [--module ] [--backup-dir ] [--branch ] [--import-webhook-status disable|current]` @@ -131,6 +132,7 @@ FLAGS --replace-existing Replaces the existing module in the target stack. --skip-app-recreation [optional] Skip private apps recreation if already exist + --skip-audit Skips the audit fix. --skip-existing Skips the module exists warning messages. DESCRIPTION @@ -154,7 +156,7 @@ EXAMPLES $ csdx cm:stacks:import --alias --config - $ csdx cm:stacks:import --branch --yes + $ csdx cm:stacks:import --branch --yes --skip-audit ``` _See code: [src/commands/cm/stacks/import.ts](https://github.com/contentstack/cli/blob/main/packages/contentstack-import/src/commands/cm/stacks/import.ts)_ diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index 35c212e1b1..185e5a6515 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -1,15 +1,15 @@ { "name": "@contentstack/cli-cm-import", "description": "Contentstack CLI plugin to import content into stack", - "version": "1.12.2", + "version": "1.13.0", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { + "@contentstack/cli-audit": "^1.3.2", "@contentstack/cli-command": "~1.2.16", "@contentstack/cli-utilities": "~1.5.10", "@contentstack/management": "~1.13.0", "@oclif/core": "^2.9.3", - "axios": "^1.6.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", "chalk": "^4.1.2", @@ -99,4 +99,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} \ No newline at end of file +} diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index fbf94f43b6..047a528453 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -9,7 +9,6 @@ import { ContentstackClient, } from '@contentstack/cli-utilities'; -import { trace } from '../../../utils/log'; import { ImportConfig } from '../../../types'; import { ModuleImporter } from '../../../import'; import { setupImportConfig, formatError, log } from '../../../utils'; @@ -25,7 +24,7 @@ export default class ImportCommand extends Command { `csdx cm:stacks:import --alias `, `csdx cm:stacks:import --alias --data-dir `, `csdx cm:stacks:import --alias --config `, - `csdx cm:stacks:import --branch --yes`, + `csdx cm:stacks:import --branch --yes --skip-audit`, ]; static flags: FlagInput = { @@ -106,6 +105,9 @@ export default class ImportCommand extends Command { default: false, description: 'Skips the module exists warning messages.', }), + 'skip-audit': flags.boolean({ + description: 'Skips the audit fix.', + }), }; static aliases: string[] = ['cm:import']; @@ -127,8 +129,12 @@ export default class ImportCommand extends Command { const managementAPIClient: ContentstackClient = await managementSDKClient(importConfig); const moduleImporter = new ModuleImporter(managementAPIClient, importConfig); - await moduleImporter.start(); - log(importConfig, `The content has been imported to the stack ${importConfig.apiKey} successfully!`, 'success'); + const result = await moduleImporter.start(); + + if (!result?.noSuccessMsg) { + log(importConfig, `The content has been imported to the stack ${importConfig.apiKey} successfully!`, 'success'); + } + log( importConfig, `The log has been stored at '${path.join(importConfig.backupDir, 'logs', 'import')}'`, diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index f92999429b..1fd509978e 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -1,9 +1,12 @@ -import { addLocale, ContentstackClient } from '@contentstack/cli-utilities'; +import { resolve } from 'path'; +import { AuditFix } from '@contentstack/cli-audit'; +import messages, { $t } from '@contentstack/cli-audit/lib/messages'; +import { addLocale, cliux, ContentstackClient, Logger } from '@contentstack/cli-utilities'; import startModuleImport from './modules'; import startJSModuleImport from './modules-js'; import { ImportConfig, Modules } from '../types'; -import { backupHandler, log, validateBranch, masterLocalDetails, sanitizeStack, initLogger } from '../utils'; +import { backupHandler, log, validateBranch, masterLocalDetails, sanitizeStack, initLogger, trace } from '../utils'; class ModuleImporter { private managementAPIClient: ContentstackClient; @@ -24,18 +27,10 @@ class ModuleImporter { await validateBranch(this.stackAPIClient, this.importConfig, this.importConfig.branchName); } - // Temporarily adding this api call to verify management token has read and write permissions - // TODO: CS-40354 - CLI | import rewrite | Migrate HTTP call to SDK call once fix is ready from SDK side - if (this.importConfig.management_token) { await addLocale(this.importConfig.apiKey, this.importConfig.management_token, this.importConfig.host); } - if (!this.importConfig.master_locale) { - let masterLocalResponse = await masterLocalDetails(this.stackAPIClient); - this.importConfig['master_locale'] = { code: masterLocalResponse.code }; - this.importConfig.masterLocale = { code: masterLocalResponse.code }; - } const backupDir = await backupHandler(this.importConfig); if (backupDir) { this.importConfig.backupDir = backupDir; @@ -44,7 +39,24 @@ class ModuleImporter { } // NOTE init log - initLogger(this.importConfig); + const logger = initLogger(this.importConfig); + + // NOTE audit and fix the import content. + if ( + !this.importConfig.skipAudit && + (!this.importConfig.moduleName || + ['content-types', 'global-fields', 'entries'].includes(this.importConfig.moduleName)) + ) { + if (!(await this.auditImportData(logger))) { + return { noSuccessMsg: true }; + } + } + + if (!this.importConfig.master_locale) { + let masterLocalResponse = await masterLocalDetails(this.stackAPIClient); + this.importConfig['master_locale'] = { code: masterLocalResponse.code }; + this.importConfig.masterLocale = { code: masterLocalResponse.code }; + } await sanitizeStack(this.stackAPIClient); @@ -90,6 +102,67 @@ class ModuleImporter { await this.importByModuleByName(moduleName); } } + + /** + * The `auditImportData` function performs an audit process on imported data, using a specified + * configuration, and returns a boolean indicating whether a fix is needed. + * @returns The function `auditImportData()` returns a boolean value. It returns `true` if there is a + * fix available and the user confirms to proceed with the fix, otherwise it returns `false`. + */ + async auditImportData(logger: Logger) { + const basePath = resolve(this.importConfig.backupDir, 'logs', 'audit'); + const auditConfig = { + noLog: false, // Skip logs printing on terminal + skipConfirm: true, // Skip confirmation if any + returnResponse: true, // On process completion should return config used in the command + noTerminalOutput: false, // Skip final audit table output on terminal + config: { basePath }, // To overwrite any build-in config. This config is equal to --config flag. + }; + try { + const args = [ + '--data-dir', + this.importConfig.backupDir, + '--external-config', + JSON.stringify(auditConfig), + '--report-path', + basePath, + ]; + + if (this.importConfig.moduleName) { + args.push('--modules', this.importConfig.moduleName); + } + + log(this.importConfig, 'Starting audit process', 'info'); + const result = await AuditFix.run(args); + log(this.importConfig, 'Audit process completed', 'info'); + + if (result) { + const { hasFix, config } = result; + + if (hasFix) { + logger.log($t(messages.FINAL_REPORT_PATH, { path: config.reportPath }), 'warn'); + + if ( + this.importConfig.forceStopMarketplaceAppsPrompt || + (await cliux.inquire({ + type: 'confirm', + name: 'confirmation', + message: 'Can you check the fix on the given path and confirm if you would like to proceed with the fix?', + })) + ) { + return true; + } + + return false; + } + } + + return true; + } catch (error) { + trace(error); + log(this.importConfig, `Audit failed with following error. ${error}`, 'error'); + } + } } export default ModuleImporter; diff --git a/packages/contentstack-import/src/types/import-config.ts b/packages/contentstack-import/src/types/import-config.ts index 036dccb0a5..bdf75c4727 100644 --- a/packages/contentstack-import/src/types/import-config.ts +++ b/packages/contentstack-import/src/types/import-config.ts @@ -47,6 +47,7 @@ export default interface ImportConfig extends DefaultConfig, ExternalConfig { contentVersion: number; replaceExisting?: boolean; skipExisting?: boolean; + skipAudit?: boolean; } type branch = { diff --git a/packages/contentstack-import/src/utils/import-config-handler.ts b/packages/contentstack-import/src/utils/import-config-handler.ts index b83fd00951..1421a3e5c6 100644 --- a/packages/contentstack-import/src/utils/import-config-handler.ts +++ b/packages/contentstack-import/src/utils/import-config-handler.ts @@ -64,8 +64,9 @@ const setupConfig = async (importCmdFlags: any): Promise => { //Note to support the old key config.source_stack = config.apiKey; - config.importWebhookStatus = importCmdFlags['import-webhook-status']; + config.skipAudit = importCmdFlags['skip-audit']; config.forceStopMarketplaceAppsPrompt = importCmdFlags.yes; + config.importWebhookStatus = importCmdFlags['import-webhook-status']; config.skipPrivateAppRecreationIfExist = importCmdFlags['skip-app-recreation']; if (importCmdFlags['branch']) { diff --git a/packages/contentstack-import/src/utils/log.ts b/packages/contentstack-import/src/utils/log.ts index 9b5e3775b7..ead46c0964 100644 --- a/packages/contentstack-import/src/utils/log.ts +++ b/packages/contentstack-import/src/utils/log.ts @@ -33,4 +33,6 @@ export function initLogger(config?: ImportConfig | undefined) { return logger; } +export { logger }; + export const trace = log; diff --git a/packages/contentstack-seed/package.json b/packages/contentstack-seed/package.json index 9b6f1ea5ec..a5998b0e1f 100644 --- a/packages/contentstack-seed/package.json +++ b/packages/contentstack-seed/package.json @@ -1,11 +1,11 @@ { "name": "@contentstack/cli-cm-seed", "description": "create a Stack from existing content types, entries, assets, etc.", - "version": "1.7.0", + "version": "1.7.1", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { - "@contentstack/cli-cm-import": "~1.12.2", + "@contentstack/cli-cm-import": "~1.13.0", "@contentstack/cli-command": "~1.2.16", "@contentstack/cli-utilities": "~1.5.10", "inquirer": "8.2.4", @@ -22,7 +22,7 @@ "@types/node": "^14.14.32", "@types/tar": "^6.1.3", "@types/tmp": "^0.2.0", - "axios": "^1.6.3", + "axios": "^1.6.4", "eslint": "^8.18.0", "eslint-config-oclif": "^4.0.0", "eslint-config-oclif-typescript": "^3.0.8", @@ -73,4 +73,4 @@ "version": "oclif readme && git add README.md", "clean": "rm -rf ./node_modules tsconfig.build.tsbuildinfo" } -} \ No newline at end of file +} diff --git a/packages/contentstack-seed/src/seed/importer.ts b/packages/contentstack-seed/src/seed/importer.ts index daf507a901..930c42090a 100644 --- a/packages/contentstack-seed/src/seed/importer.ts +++ b/packages/contentstack-seed/src/seed/importer.ts @@ -22,5 +22,5 @@ export async function run(options: ImporterOptions) { : ['-k', options.api_key, '-d', importPath]; process.chdir(options.tmpPath); - await ImportCommand.run(args); + await ImportCommand.run(args.concat('--skip-audit')); } diff --git a/packages/contentstack-utilities/package.json b/packages/contentstack-utilities/package.json index e44f9b07e2..71b0004de0 100644 --- a/packages/contentstack-utilities/package.json +++ b/packages/contentstack-utilities/package.json @@ -35,7 +35,7 @@ "@contentstack/management": "~1.13.0", "@contentstack/marketplace-sdk": "^1.0.1", "@oclif/core": "^2.9.3", - "axios": "^1.6.3", + "axios": "^1.6.4", "chalk": "^4.0.0", "cli-cursor": "^3.1.0", "cli-table": "^0.3.11", @@ -83,4 +83,4 @@ "tslib": "^1.13.0", "typescript": "^4.9.3" } -} \ No newline at end of file +} diff --git a/packages/contentstack/package.json b/packages/contentstack/package.json index fcaf9b5ee6..76665c1d8d 100755 --- a/packages/contentstack/package.json +++ b/packages/contentstack/package.json @@ -25,14 +25,14 @@ "@contentstack/cli-audit": "~1.3.2", "@contentstack/cli-auth": "~1.3.17", "@contentstack/cli-cm-bootstrap": "~1.8.0", - "@contentstack/cli-cm-branches": "~1.0.19", + "@contentstack/cli-cm-branches": "~1.0.20", "@contentstack/cli-cm-bulk-publish": "~1.4.0", - "@contentstack/cli-cm-clone": "~1.8.0", + "@contentstack/cli-cm-clone": "~1.9.0", "@contentstack/cli-cm-export": "~1.10.2", "@contentstack/cli-cm-export-to-csv": "~1.6.2", - "@contentstack/cli-cm-import": "~1.12.2", + "@contentstack/cli-cm-import": "~1.13.0", "@contentstack/cli-cm-migrate-rte": "~1.4.15", - "@contentstack/cli-cm-seed": "~1.7.0", + "@contentstack/cli-cm-seed": "~1.7.1", "@contentstack/cli-command": "~1.2.17", "@contentstack/cli-config": "~1.5.1", "@contentstack/cli-launch": "~1.0.15", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 592c138b6c..345caf9f8c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,14 +13,14 @@ importers: '@contentstack/cli-audit': ~1.3.2 '@contentstack/cli-auth': ~1.3.17 '@contentstack/cli-cm-bootstrap': ~1.8.0 - '@contentstack/cli-cm-branches': ~1.0.19 + '@contentstack/cli-cm-branches': ~1.0.20 '@contentstack/cli-cm-bulk-publish': ~1.4.0 - '@contentstack/cli-cm-clone': ~1.8.0 + '@contentstack/cli-cm-clone': ~1.9.0 '@contentstack/cli-cm-export': ~1.10.2 '@contentstack/cli-cm-export-to-csv': ~1.6.2 - '@contentstack/cli-cm-import': ~1.12.2 + '@contentstack/cli-cm-import': ~1.13.0 '@contentstack/cli-cm-migrate-rte': ~1.4.15 - '@contentstack/cli-cm-seed': ~1.7.0 + '@contentstack/cli-cm-seed': ~1.7.1 '@contentstack/cli-command': ~1.2.17 '@contentstack/cli-config': ~1.5.1 '@contentstack/cli-launch': ~1.0.15 @@ -422,7 +422,7 @@ importers: specifiers: '@colors/colors': ^1.5.0 '@contentstack/cli-cm-export': ~1.10.2 - '@contentstack/cli-cm-import': ~1.12.2 + '@contentstack/cli-cm-import': ~1.13.0 '@contentstack/cli-command': ~1.2.16 '@contentstack/cli-utilities': ~1.5.10 '@oclif/test': ^1.2.7 @@ -724,6 +724,7 @@ importers: packages/contentstack-import: specifiers: + '@contentstack/cli-audit': ^1.3.2 '@contentstack/cli-command': ~1.2.16 '@contentstack/cli-utilities': ~1.5.10 '@contentstack/management': ~1.13.0 @@ -740,7 +741,6 @@ importers: '@types/tar': ^4.0.3 '@types/uuid': ^9.0.7 '@typescript-eslint/eslint-plugin': ^5.48.2 - axios: ^1.6.3 big-json: ^3.2.0 bluebird: ^3.7.2 chai: ^4.2.0 @@ -767,11 +767,11 @@ importers: uuid: ^9.0.0 winston: ^3.7.2 dependencies: + '@contentstack/cli-audit': link:../contentstack-audit '@contentstack/cli-command': link:../contentstack-command '@contentstack/cli-utilities': link:../contentstack-utilities '@contentstack/management': 1.13.0_debug@4.3.4 '@oclif/core': 2.15.0_ogreqof3k35xezedraj6pnd45y - axios: 1.6.3_debug@4.3.4 big-json: 3.2.0 bluebird: 3.7.2 chalk: 4.1.2 @@ -984,7 +984,7 @@ importers: packages/contentstack-seed: specifiers: - '@contentstack/cli-cm-import': ~1.12.2 + '@contentstack/cli-cm-import': ~1.13.0 '@contentstack/cli-command': ~1.2.16 '@contentstack/cli-utilities': ~1.5.10 '@oclif/plugin-help': ^5.1.19 @@ -994,7 +994,7 @@ importers: '@types/node': ^14.14.32 '@types/tar': ^6.1.3 '@types/tmp': ^0.2.0 - axios: ^1.6.3 + axios: ^1.6.4 eslint: ^8.18.0 eslint-config-oclif: ^4.0.0 eslint-config-oclif-typescript: ^3.0.8 @@ -1026,7 +1026,7 @@ importers: '@types/node': 14.18.63 '@types/tar': 6.1.10 '@types/tmp': 0.2.6 - axios: 1.6.3 + axios: 1.6.5 eslint: 8.55.0 eslint-config-oclif: 4.0.0_eslint@8.55.0 eslint-config-oclif-typescript: 3.0.26_sjjl3gun7puonkp27uqtyjm5b4 @@ -1051,7 +1051,7 @@ importers: '@types/node': ^14.14.32 '@types/sinon': ^10.0.2 '@types/traverse': ^0.6.34 - axios: ^1.6.3 + axios: ^1.6.4 chai: ^4.3.4 chalk: ^4.0.0 cli-cursor: ^3.1.0 @@ -1091,7 +1091,7 @@ importers: '@contentstack/management': 1.13.0_debug@4.3.4 '@contentstack/marketplace-sdk': 1.0.1_debug@4.3.4 '@oclif/core': 2.15.0_ogreqof3k35xezedraj6pnd45y - axios: 1.6.3_debug@4.3.4 + axios: 1.6.5_debug@4.3.4 chalk: 4.1.2 cli-cursor: 3.1.0 cli-table: 0.3.11 @@ -1607,7 +1607,7 @@ packages: '@contentstack/management': 1.13.0_debug@4.3.4 '@contentstack/marketplace-sdk': 1.0.1_debug@4.3.4 '@oclif/core': 2.15.0_typescript@4.9.5 - axios: 1.6.3_debug@4.3.4 + axios: 1.6.5_debug@4.3.4 chalk: 4.1.2 cli-cursor: 3.1.0 cli-table: 0.3.11 @@ -1661,7 +1661,7 @@ packages: /@contentstack/marketplace-sdk/1.0.1_debug@4.3.4: resolution: {integrity: sha512-E9amU6qwp4i1AzXiMX4UKYJSZ8wyhncGPx5MGpwGoM2WUfJqGMwH9o1bXLAzOSBu1a8yhOMOkz/ehLwYeq6Ufg==} dependencies: - axios: 1.6.3_debug@4.3.4 + axios: 1.6.5_debug@4.3.4 transitivePeerDependencies: - debug @@ -4758,20 +4758,20 @@ packages: transitivePeerDependencies: - debug - /axios/1.6.3: - resolution: {integrity: sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==} + /axios/1.6.5: + resolution: {integrity: sha512-Ii012v05KEVuUoFWmMW/UQv9aRIc3ZwkWDcM+h5Il8izZCtRVpDUfwpoFf7eOtajT3QiGR4yDUx7lPqHJULgbg==} dependencies: - follow-redirects: 1.15.3 + follow-redirects: 1.15.4 form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug dev: true - /axios/1.6.3_debug@4.3.4: - resolution: {integrity: sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==} + /axios/1.6.5_debug@4.3.4: + resolution: {integrity: sha512-Ii012v05KEVuUoFWmMW/UQv9aRIc3ZwkWDcM+h5Il8izZCtRVpDUfwpoFf7eOtajT3QiGR4yDUx7lPqHJULgbg==} dependencies: - follow-redirects: 1.15.3_debug@4.3.4 + follow-redirects: 1.15.4_debug@4.3.4 form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -7498,9 +7498,20 @@ packages: /fn.name/1.1.0: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} - /follow-redirects/1.15.3: + /follow-redirects/1.15.3_debug@4.3.4: resolution: {integrity: sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==} engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dependencies: + debug: 4.3.4 + + /follow-redirects/1.15.4: + resolution: {integrity: sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==} + engines: {node: '>=4.0'} peerDependencies: debug: '*' peerDependenciesMeta: @@ -7508,8 +7519,8 @@ packages: optional: true dev: true - /follow-redirects/1.15.3_debug@4.3.4: - resolution: {integrity: sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==} + /follow-redirects/1.15.4_debug@4.3.4: + resolution: {integrity: sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==} engines: {node: '>=4.0'} peerDependencies: debug: '*'