From b3c6d175f260ec822f53471270b5e9de0fe6d0d3 Mon Sep 17 00:00:00 2001 From: Antony Date: Thu, 19 Oct 2023 16:21:11 +0530 Subject: [PATCH 1/3] Fix: cm:assets:publish command not working as expected --- .../src/producer/publish-assets.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/contentstack-bulk-publish/src/producer/publish-assets.js b/packages/contentstack-bulk-publish/src/producer/publish-assets.js index 698855e067..1846b13aff 100644 --- a/packages/contentstack-bulk-publish/src/producer/publish-assets.js +++ b/packages/contentstack-bulk-publish/src/producer/publish-assets.js @@ -51,7 +51,7 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe environments: environments, locale, stack: stack, - apiVersion + apiVersion, }); bulkPublishSet = []; } @@ -63,7 +63,7 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe environments: environments, locale, stack: stack, - apiVersion + apiVersion, }); bulkPublishSet = []; } @@ -83,6 +83,8 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe } await getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, skip); return resolve(); + } else { + resolve(); } }) .catch((error) => { From ce6d82e6d01934b7ed3fd8e0b54ea62db29e173a Mon Sep 17 00:00:00 2001 From: Antony Date: Mon, 23 Oct 2023 11:43:02 +0530 Subject: [PATCH 2/3] Fix: Snyk, lint issue fixes, better error logger added --- package-lock.json | 44 ++++ .../src/audit-base-command.ts | 4 +- .../contentstack-audit/src/modules/entries.ts | 75 ++++--- packages/contentstack-branches/README.md | 2 +- .../src/commands/cm/stacks/import.ts | 4 + .../src/import/module-importer.ts | 11 +- .../src/import/modules-js/marketplace-apps.js | 16 +- .../src/import/modules/marketplace-apps.ts | 15 +- .../src/utils/backup-handler.ts | 72 +++--- .../contentstack-import/src/utils/index.ts | 1 + packages/contentstack-import/src/utils/log.ts | 36 +++ .../contentstack-import/src/utils/logger.ts | 4 +- .../src/utils/marketplace-app-helper.ts | 23 +- .../src/commands/launch/base-command.ts | 14 +- packages/contentstack-utilities/package.json | 5 +- packages/contentstack-utilities/src/index.ts | 3 + packages/contentstack-utilities/src/logger.ts | 207 +++++++++++++++++- pnpm-lock.yaml | 19 ++ 18 files changed, 448 insertions(+), 107 deletions(-) create mode 100644 packages/contentstack-import/src/utils/log.ts diff --git a/package-lock.json b/package-lock.json index 3d69b66d7f..000fcf7602 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2898,6 +2898,12 @@ "integrity": "sha512-dDZH/tXzwjutnuk4UacGgFRwV+JSLaXL1ikvidfJprkb7L9Nx1njcRHHmi3Dsvt7pgqqTEeucQuOrWHPFgzVHA==", "dev": true }, + "node_modules/@types/traverse": { + "version": "0.6.34", + "resolved": "https://registry.npmjs.org/@types/traverse/-/traverse-0.6.34.tgz", + "integrity": "sha512-bTvEo1/mx4ERGCcZfcvI8hBIUulNhqMcgyX/JzZ+p0WGdiU+oVjIJJJUs5zrkeWpfrTj98UvUm0qVHUk5lnkUg==", + "dev": true + }, "node_modules/@types/triple-beam": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", @@ -11519,6 +11525,14 @@ "node": ">=6" } }, + "node_modules/klona": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/klona/-/klona-2.0.6.tgz", + "integrity": "sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==", + "engines": { + "node": ">= 8" + } + }, "node_modules/kuler": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", @@ -20514,6 +20528,14 @@ "node": ">=12" } }, + "node_modules/traverse": { + "version": "0.6.7", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.7.tgz", + "integrity": "sha512-/y956gpUo9ZNCb99YjxG7OaslxZWHfCHAUUfshwqOXmxUIvqLjVO581BT+gM59+QV9tFe6/CGG53tsA1Y7RSdg==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", @@ -25253,11 +25275,13 @@ "inquirer": "8.2.4", "inquirer-search-checkbox": "^1.0.0", "inquirer-search-list": "^1.2.6", + "klona": "^2.0.6", "lodash": "^4.17.15", "mkdirp": "^1.0.4", "open": "^8.4.2", "ora": "^5.4.0", "rxjs": "^6.6.7", + "traverse": "^0.6.7", "unique-string": "^2.0.0", "uuid": "^9.0.0", "winston": "^3.7.2", @@ -25271,6 +25295,7 @@ "@types/mocha": "^8.2.2", "@types/node": "^14.14.32", "@types/sinon": "^10.0.2", + "@types/traverse": "^0.6.34", "chai": "^4.3.4", "eslint": "^8.18.0", "eslint-config-oclif": "^4.0.0", @@ -28738,6 +28763,7 @@ "@types/mocha": "^8.2.2", "@types/node": "^14.14.32", "@types/sinon": "^10.0.2", + "@types/traverse": "^0.6.34", "axios": "1.3.4", "chai": "^4.3.4", "chalk": "^4.0.0", @@ -28754,6 +28780,7 @@ "inquirer": "8.2.4", "inquirer-search-checkbox": "^1.0.0", "inquirer-search-list": "^1.2.6", + "klona": "^2.0.6", "lodash": "^4.17.15", "mkdirp": "^1.0.4", "mocha": "10.1.0", @@ -28765,6 +28792,7 @@ "rxjs": "^6.6.7", "sinon": "^15.0.1", "tmp": "^0.2.1", + "traverse": "^0.6.7", "ts-node": "^10.9.1", "tslib": "^1.13.0", "typescript": "^4.9.3", @@ -31382,6 +31410,12 @@ "integrity": "sha512-dDZH/tXzwjutnuk4UacGgFRwV+JSLaXL1ikvidfJprkb7L9Nx1njcRHHmi3Dsvt7pgqqTEeucQuOrWHPFgzVHA==", "dev": true }, + "@types/traverse": { + "version": "0.6.34", + "resolved": "https://registry.npmjs.org/@types/traverse/-/traverse-0.6.34.tgz", + "integrity": "sha512-bTvEo1/mx4ERGCcZfcvI8hBIUulNhqMcgyX/JzZ+p0WGdiU+oVjIJJJUs5zrkeWpfrTj98UvUm0qVHUk5lnkUg==", + "dev": true + }, "@types/triple-beam": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", @@ -37835,6 +37869,11 @@ "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", "dev": true }, + "klona": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/klona/-/klona-2.0.6.tgz", + "integrity": "sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==" + }, "kuler": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", @@ -44445,6 +44484,11 @@ "punycode": "^2.1.1" } }, + "traverse": { + "version": "0.6.7", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.7.tgz", + "integrity": "sha512-/y956gpUo9ZNCb99YjxG7OaslxZWHfCHAUUfshwqOXmxUIvqLjVO581BT+gM59+QV9tFe6/CGG53tsA1Y7RSdg==" + }, "tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", diff --git a/packages/contentstack-audit/src/audit-base-command.ts b/packages/contentstack-audit/src/audit-base-command.ts index 7cc3fb1f60..bb3f2262bb 100644 --- a/packages/contentstack-audit/src/audit-base-command.ts +++ b/packages/contentstack-audit/src/audit-base-command.ts @@ -176,8 +176,8 @@ export abstract class AuditBaseCommand extends BaseCommand {}, - config: this.config, - moduleName: 'content-types', - ctSchema: this.ctSchema, - gfSchema: this.gfSchema, - }).run(true)) as ContentTypeStruct[]; - this.gfSchema = (await new ContentType({ - fix: true, - log: () => {}, - config: this.config, - moduleName: 'entries', - ctSchema: this.ctSchema, - gfSchema: this.gfSchema, - }).run(true)) as ContentTypeStruct[]; + await this.fixPrerequisiteData(); for (const { code } of this.locales) { for (const ctSchema of this.ctSchema) { @@ -125,13 +109,43 @@ export default class Entries { } this.log('', 'info'); // Adding empty line + this.removeEmptyVal(); + + return this.missingRefs; + } + + /** + * The function removes any properties from the `missingRefs` object that have an empty array value. + */ + removeEmptyVal() { for (let propName in this.missingRefs) { if (!this.missingRefs[propName].length) { delete this.missingRefs[propName]; } } + } - return this.missingRefs; + /** + * The function `fixPrerequisiteData` fixes the prerequisite data by updating the `ctSchema` and + * `gfSchema` properties using the `ContentType` class. + */ + async fixPrerequisiteData() { + this.ctSchema = (await new ContentType({ + fix: true, + log: () => {}, + config: this.config, + moduleName: 'content-types', + ctSchema: this.ctSchema, + gfSchema: this.gfSchema, + }).run(true)) as ContentTypeStruct[]; + this.gfSchema = (await new ContentType({ + fix: true, + log: () => {}, + config: this.config, + moduleName: 'entries', + ctSchema: this.ctSchema, + gfSchema: this.gfSchema, + }).run(true)) as ContentTypeStruct[]; } /** @@ -449,13 +463,14 @@ export default class Entries { if (data_type === 'json') { if (field.field_metadata.extension) { // NOTE Custom field type - return field; + break; } else if (field.field_metadata.allow_json_rte) { - return this.fixJsonRteMissingReferences( + this.fixJsonRteMissingReferences( [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], field as JsonRTEFieldDataType, entry[uid] as EntryJsonRTEFieldDataType, ); + break; } } @@ -464,7 +479,7 @@ export default class Entries { [...tree, { uid: field.uid, name: field.display_name, data_type: field.data_type }], field as ReferenceFieldDataType, entry[uid] as EntryReferenceFieldDataType[], - ) as EntryReferenceFieldDataType[]; + ); break; case 'blocks': entry[uid] = this.fixModularBlocksReferences( @@ -541,7 +556,7 @@ export default class Entries { return eBlock; }) - .filter((val) => !isEmpty(val)) as EntryModularBlocksDataType[]; + .filter((val) => !isEmpty(val)); }); return entry; @@ -604,11 +619,7 @@ export default class Entries { ) { if (Array.isArray(entry)) { entry = entry.map((child: any, index) => { - return this.fixJsonRteMissingReferences( - [...tree, { index, type: (child as any)?.type, uid: child?.uid }], - field, - child, - ); + return this.fixJsonRteMissingReferences([...tree, { index, type: child?.type, uid: child?.uid }], field, child); }) as EntryJsonRTEFieldDataType[]; } else { entry.children = entry.children @@ -669,7 +680,7 @@ export default class Entries { data_type: field.data_type, display_name: field.display_name, treeStr: tree - .map(({ name, index }) => (index || index === 0 ? `[${index}].${name}` : name)) + .map(({ name, index }) => (index || index === 0 ? `[${+index}].${name}` : name)) .filter((val) => val) .join(' ➜ '), missingRefs, @@ -697,7 +708,7 @@ export default class Entries { tree: Record[], blocks: ModularBlockType[], entryBlock: EntryModularBlocksDataType, - index: Number, + index: number, ) { const validBlockUid = blocks.map((block) => block.uid); const invalidKeys = Object.keys(entryBlock).filter((key) => !validBlockUid.includes(key)); @@ -715,7 +726,7 @@ export default class Entries { fixStatus: this.fix ? 'Fixed' : undefined, tree: [...tree, { index, uid: key, name: key }], treeStr: [...tree, { index, uid: key, name: key }] - .map(({ name, index }) => (index || index === 0 ? `[${index}].${name}` : name)) + .map(({ name, index }) => (index || index === 0 ? `[${+index}].${name}` : name)) .filter((val) => val) .join(' ➜ '), missingRefs: [key], @@ -775,8 +786,8 @@ export default class Entries { const localesFolderPath = resolve(this.config.basePath, this.config.moduleConfig.locales.dirName); const localesPath = join(localesFolderPath, this.config.moduleConfig.locales.fileName); const masterLocalesPath = join(localesFolderPath, 'master-locale.json'); - this.locales = values(JSON.parse(readFileSync(masterLocalesPath, 'utf-8'))); - this.locales.push(...values(JSON.parse(readFileSync(localesPath, 'utf-8')))); + this.locales = values(JSON.parse(readFileSync(masterLocalesPath, 'utf8'))); + this.locales.push(...values(JSON.parse(readFileSync(localesPath, 'utf8')))); for (const { code } of this.locales) { for (const { uid } of this.ctSchema) { diff --git a/packages/contentstack-branches/README.md b/packages/contentstack-branches/README.md index 023492364b..9587da605b 100755 --- a/packages/contentstack-branches/README.md +++ b/packages/contentstack-branches/README.md @@ -37,7 +37,7 @@ $ npm install -g @contentstack/cli-cm-branches $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-branches/1.0.15 darwin-arm64 node-v20.3.1 +@contentstack/cli-cm-branches/1.0.15 darwin-arm64 node-v20.8.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index 64f2d73e7a..45bf0ccc3a 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -8,6 +8,8 @@ import { FlagInput, ContentstackClient, } from '@contentstack/cli-utilities'; + +import { trace } from '../../../utils/log'; import { ImportConfig } from '../../../types'; import { ModuleImporter } from '../../../import'; import { setupImportConfig, formatError, log } from '../../../utils'; @@ -120,6 +122,7 @@ export default class ImportCommand extends Command { // Note setting host to create cma client importConfig.host = this.cmaHost; backupDir = importConfig.backupDir; + const managementAPIClient: ContentstackClient = await managementSDKClient(importConfig); const moduleImporter = new ModuleImporter(managementAPIClient, importConfig); await moduleImporter.start(); @@ -130,6 +133,7 @@ export default class ImportCommand extends Command { 'success', ); } catch (error) { + trace(error, 'error', true); log({ data: backupDir } as ImportConfig, `Failed to import stack content - ${formatError(error)}`, 'error'); log( { data: backupDir } as ImportConfig, diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index 9018fe8cf5..293616e483 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -1,8 +1,9 @@ import { ContentstackClient, HttpClient } from '@contentstack/cli-utilities'; -import { ImportConfig, Modules } from '../types'; -import { backupHandler, log, validateBranch, masterLocalDetails, sanitizeStack } from '../utils'; + import startModuleImport from './modules'; import startJSModuleImport from './modules-js'; +import { ImportConfig, Modules } from '../types'; +import { backupHandler, log, validateBranch, masterLocalDetails, sanitizeStack, initLogger } from '../utils'; class ModuleImporter { private managementAPIClient: ContentstackClient; @@ -25,7 +26,6 @@ class ModuleImporter { // Temporarily adding this api call to verify management token has read and write permissions // TODO: CS-40354 - CLI | import rewrite | Migrate HTTP call to SDK call once fix is ready from SDK side - const httpClient = new HttpClient({ headers: { api_key: this.importConfig.apiKey, authorization: this.importConfig.management_token }, }); @@ -53,6 +53,9 @@ class ModuleImporter { this.importConfig.data = backupDir; } + // NOTE init log + initLogger(this.importConfig); + await sanitizeStack(this.stackAPIClient); return this.import(); @@ -72,7 +75,7 @@ class ModuleImporter { log(this.importConfig, `Starting import of ${moduleName} module`, 'info'); // import the modules by name // calls the module runner which inturn calls the module itself - // Todo: Implement a mechanism to determine whether module is new or old + // NOTE: Implement a mechanism to determine whether module is new or old if (this.importConfig.contentVersion === 2) { return startModuleImport({ stackAPIClient: this.stackAPIClient, diff --git a/packages/contentstack-import/src/import/modules-js/marketplace-apps.js b/packages/contentstack-import/src/import/modules-js/marketplace-apps.js index f0af583cc8..a6cc230857 100644 --- a/packages/contentstack-import/src/import/modules-js/marketplace-apps.js +++ b/packages/contentstack-import/src/import/modules-js/marketplace-apps.js @@ -17,13 +17,16 @@ const { HttpClientDecorator, OauthDecorator, } = require('@contentstack/cli-utilities'); + const { log, - fileHelper: { readFileSync, writeFile }, formatError, + fileHelper: { readFileSync, writeFile }, } = require('../../utils'); -let { default: config } = require('../../config'); +const { trace } = require('../../utils/log'); +const { default: config } = require('../../config'); const { getDeveloperHubUrl, getAllStackSpecificApps } = require('../../utils/marketplace-app-helper'); + module.exports = class ImportMarketplaceApps { client; httpClient; @@ -409,6 +412,7 @@ module.exports = class ImportMarketplaceApps { this.installationUidMapping[app.uid] = installation.installation_uid; updateParam = { manifest: app.manifest, ...installation, configuration, server_configuration }; } else if (installation.message) { + trace(installation, 'error', true); log(this.config, formatError(installation.message), 'success'); await this.confirmToCloseProcess(installation); } @@ -434,6 +438,7 @@ module.exports = class ImportMarketplaceApps { .get(response.redirect_url) .then(async ({ response }) => { if (_.includes([501, 403], response.status)) { + trace(response, 'error', true); // NOTE Log complete stack and hide on UI log(this.config, `${appName} - ${response.statusText}, OAuth api call failed.!`, 'error'); log(this.config, formatError(response), 'error'); await this.confirmToCloseProcess({ message: response.data }); @@ -442,6 +447,7 @@ module.exports = class ImportMarketplaceApps { } }) .catch((error) => { + trace(error, 'error', true); if (_.includes([501, 403], error.status)) { log(this.config, formatError(error), 'error'); } @@ -529,12 +535,16 @@ module.exports = class ImportMarketplaceApps { }) .then(({ data }) => { if (data.message) { + trace(data, 'error', true); log(this.config, formatError(data.message), 'success'); } else { log(this.config, `${app.manifest.name} app config updated successfully.!`, 'success'); } }) - .catch((error) => log(this.config, formatError(error), 'error')); + .catch((error) => { + trace(data, 'error', true); + log(this.config, formatError(error), 'error') + }); } validateAppName(name) { diff --git a/packages/contentstack-import/src/import/modules/marketplace-apps.ts b/packages/contentstack-import/src/import/modules/marketplace-apps.ts index 63869e6cc7..ca36361539 100644 --- a/packages/contentstack-import/src/import/modules/marketplace-apps.ts +++ b/packages/contentstack-import/src/import/modules/marketplace-apps.ts @@ -19,8 +19,8 @@ import { NodeCrypto, ContentstackClient, } from '@contentstack/cli-utilities'; - import BaseClass from './base-class'; +import { trace } from '../../utils/log'; import { askEncryptionKey } from '../../utils/interactive'; import { ModuleClassParams, MarketplaceAppsConfig } from '../../types'; import { @@ -55,7 +55,7 @@ export default class ImportMarketplaceApps extends BaseClass { public developerHubBaseUrl: string; public sdkClient: ContentstackClient; public nodeCrypto: NodeCrypto; - public appSdkAxiosInstance: any + public appSdkAxiosInstance: any; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -101,7 +101,7 @@ export default class ImportMarketplaceApps extends BaseClass { this.developerHubBaseUrl = this.importConfig.developerHubBaseUrl || (await getDeveloperHubUrl(this.importConfig)); this.sdkClient = await managementSDKClient({ endpoint: this.developerHubBaseUrl }); this.appSdkAxiosInstance = await managementSDKClient({ - host: this.developerHubBaseUrl.split('://').pop() + host: this.developerHubBaseUrl.split('://').pop(), }); this.importConfig.org_uid = await getOrgUid(this.importConfig); await this.setHttpClient(); @@ -313,6 +313,7 @@ export default class ImportMarketplaceApps extends BaseClass { const updatedApp = await handleNameConflict(app, appSuffix, this.importConfig); return this.createPrivateApps(updatedApp, true, appSuffix + 1); } else { + trace(response, 'error', true); log(this.importConfig, formatError(message), 'error'); if (this.importConfig.forceStopMarketplaceAppsPrompt) return Promise.resolve(); @@ -412,13 +413,17 @@ export default class ImportMarketplaceApps extends BaseClass { organization_uid: this.importConfig.org_uid, }, }) - .then(({data}:any) => { + .then(({ data }: any) => { if (data?.message) { + trace(data, 'error', true); log(this.importConfig, formatError(data.message), 'success'); } else { log(this.importConfig, `${app.manifest.name} app config updated successfully.!`, 'success'); } }) - .catch((error:any) => log(this.importConfig, formatError(error), 'error')); + .catch((error: any) => { + trace(error, 'error', true); + log(this.importConfig, formatError(error), 'error'); + }); } } diff --git a/packages/contentstack-import/src/utils/backup-handler.ts b/packages/contentstack-import/src/utils/backup-handler.ts index c132a7abdc..15eda75004 100755 --- a/packages/contentstack-import/src/utils/backup-handler.ts +++ b/packages/contentstack-import/src/utils/backup-handler.ts @@ -2,50 +2,52 @@ import * as path from 'path'; import { copy } from 'fs-extra'; import { cliux } from '@contentstack/cli-utilities'; -import { fileHelper } from './index'; +import { fileHelper, trace } from './index'; import { ImportConfig } from '../types'; -export default function setupBackupDir(importConfig: ImportConfig): Promise { - return new Promise(async (resolve, reject) => { - if (importConfig.hasOwnProperty('useBackedupDir')) { - return resolve(importConfig.useBackedupDir); - } +export default async function backupHandler(importConfig: ImportConfig): Promise { + if (importConfig.hasOwnProperty('useBackedupDir')) { + return importConfig.useBackedupDir; + } - const subDir = isSubDirectory(importConfig); - let backupDirPath: string; + let backupDirPath: string; + const subDir = isSubDirectory(importConfig); - if (subDir) { - backupDirPath = path.resolve(importConfig.contentDir, '..', '_backup_' + Math.floor(Math.random() * 1000)); - if (importConfig.createBackupDir) { - cliux.print( - `Warning!!! Provided backup directory path is a sub directory of the content directory, Cannot copy to a sub directory. Hence new backup directory created - ${backupDirPath}`, - { - color: 'yellow', - }, - ); - } - } else { - //NOTE: If the backup folder's directory is provided, create it at that location; otherwise, the default path (working directory). - backupDirPath = path.join(process.cwd(), '_backup_' + Math.floor(Math.random() * 1000)); - if (importConfig.createBackupDir) { - if (fileHelper.fileExistsSync(importConfig.createBackupDir)) { - fileHelper.removeDirSync(importConfig.createBackupDir); - } - fileHelper.makeDirectory(importConfig.createBackupDir); - backupDirPath = importConfig.createBackupDir; + if (subDir) { + backupDirPath = path.resolve(importConfig.contentDir, '..', '_backup_' + Math.floor(Math.random() * 1000)); + if (importConfig.createBackupDir) { + cliux.print( + `Warning!!! Provided backup directory path is a sub directory of the content directory, Cannot copy to a sub directory. Hence new backup directory created - ${backupDirPath}`, + { + color: 'yellow', + }, + ); + } + } else { + // NOTE: If the backup folder's directory is provided, create it at that location; otherwise, the default path (working directory). + backupDirPath = path.join(process.cwd(), '_backup_' + Math.floor(Math.random() * 1000)); + if (importConfig.createBackupDir) { + if (fileHelper.fileExistsSync(importConfig.createBackupDir)) { + fileHelper.removeDirSync(importConfig.createBackupDir); } + fileHelper.makeDirectory(importConfig.createBackupDir); + backupDirPath = importConfig.createBackupDir; } + } - if (backupDirPath) { - cliux.print('Copying content to the backup directory...'); + if (backupDirPath) { + cliux.print('Copying content to the backup directory...'); + + return new Promise((resolve, reject) => { return copy(importConfig.contentDir, backupDirPath, (error: any) => { if (error) { + trace(error, 'error', true); return reject(error); } - return resolve(backupDirPath); + resolve(backupDirPath); }); - } - }); + }); + } } /** @@ -57,9 +59,11 @@ function isSubDirectory(importConfig: ImportConfig) { const parent = importConfig.contentDir; const child = importConfig.createBackupDir ? importConfig.createBackupDir : process.cwd(); const relative = path.relative(parent, child); + if (relative) { - return relative && !relative.startsWith('..') && !path.isAbsolute(relative); + return !relative.startsWith('..') && !path.isAbsolute(relative); } - //true if both parent and child have same path + + // true if both parent and child have same path return true; } diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index 19cfca611e..f0c9b9a5a1 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -27,3 +27,4 @@ export { restoreJsonRteEntryRefs, } from './entries-helper'; export * from './common-helper'; +export * from './log' \ No newline at end of file diff --git a/packages/contentstack-import/src/utils/log.ts b/packages/contentstack-import/src/utils/log.ts new file mode 100644 index 0000000000..9b5e3775b7 --- /dev/null +++ b/packages/contentstack-import/src/utils/log.ts @@ -0,0 +1,36 @@ +import { join } from 'path'; +import { LogEntry } from 'winston/index'; +import { Logger } from '@contentstack/cli-utilities'; +import { LogsType, MessageType } from '@contentstack/cli-utilities/lib/logger'; + +import { ImportConfig } from '../types'; + +let logger: Logger; + +export function isImportConfig(config: ImportConfig | MessageType): config is ImportConfig { + return (config as ImportConfig).data !== undefined && (config as ImportConfig)?.contentVersion !== undefined; +} + +export function log(entry: LogEntry): void; +export function log(error: MessageType, logType: LogsType): void; +export function log(error: MessageType, logType: 'error', hidden: boolean): void; +export function log(entryOrMessage: MessageType, logType?: LogsType, hidden?: boolean): Logger | void { + logger = initLogger(); + + if (logType === 'error') { + logger.log(entryOrMessage, logType, hidden); + } else { + logger.log(entryOrMessage, logType); + } +} + +export function initLogger(config?: ImportConfig | undefined) { + if (!logger) { + const basePath = join(config?.data ?? process.cwd(), 'logs', 'import'); + logger = new Logger(Object.assign(config ?? {}, { basePath })); + } + + return logger; +} + +export const trace = log; diff --git a/packages/contentstack-import/src/utils/logger.ts b/packages/contentstack-import/src/utils/logger.ts index d6ca174a75..2af7e031b9 100644 --- a/packages/contentstack-import/src/utils/logger.ts +++ b/packages/contentstack-import/src/utils/logger.ts @@ -58,9 +58,9 @@ let errorTransport; function init(_logPath: string) { if (!logger || !errorLogger) { - var logsDir = path.resolve(_logPath, 'logs', 'import'); + let logsDir = path.resolve(_logPath, 'logs', 'import'); // Create dir if doesn't already exist - mkdirp.sync(logsDir); + // mkdirp.sync(logsDir); successTransport = { filename: path.join(logsDir, 'success.log'), diff --git a/packages/contentstack-import/src/utils/marketplace-app-helper.ts b/packages/contentstack-import/src/utils/marketplace-app-helper.ts index 3f643472ca..f6d25bb416 100644 --- a/packages/contentstack-import/src/utils/marketplace-app-helper.ts +++ b/packages/contentstack-import/src/utils/marketplace-app-helper.ts @@ -5,10 +5,11 @@ import includes from 'lodash/includes'; import chalk from 'chalk'; import { cliux, configHandler, HttpClient, ContentstackClient, managementSDKClient } from '@contentstack/cli-utilities'; -import { ImportConfig } from '../types'; import { log } from './logger'; -import { askDeveloperHubUrl } from './interactive'; +import { trace } from '../utils/log'; +import { ImportConfig } from '../types'; import { formatError } from '../utils'; +import { askDeveloperHubUrl } from './interactive'; import { getAppName, askAppName, selectConfiguration } from '../utils/interactive'; export const getAllStackSpecificApps = async ( @@ -17,7 +18,7 @@ export const getAllStackSpecificApps = async ( config: ImportConfig, ) => { const appSdkAxiosInstance = await managementSDKClient({ - host: developerHubBaseUrl.split('://').pop() + host: developerHubBaseUrl.split('://').pop(), }); return await appSdkAxiosInstance.axiosInstance .get(`${developerHubBaseUrl}/installations?target_uids=${config.target_stack}`, { @@ -25,8 +26,11 @@ export const getAllStackSpecificApps = async ( organization_uid: config.org_uid, }, }) - .then(( {data}:any) => data.data) - .catch((error:any) => log(config, `Failed to export marketplace-apps ${formatError(error)}`, 'error')); + .then(({ data }: any) => data.data) + .catch((error: any) => { + trace(error, 'error', true); + log(config, `Failed to export marketplace-apps ${formatError(error)}`, 'error'); + }); }; export const getDeveloperHubUrl = async (config: ImportConfig): Promise => { @@ -46,6 +50,7 @@ export const getOrgUid = async (config: ImportConfig): Promise => { .stack({ api_key: config.target_stack }) .fetch() .catch((error: any) => { + trace(error, 'error', true); log(config, formatError(error), 'error'); }); @@ -124,6 +129,7 @@ export const makeRedirectUrlCall = async (response: any, appName: string, config .get(response.redirect_url) .then(async ({ response }: any) => { if (includes([501, 403], response.status)) { + trace(response, 'error', true); log(config, `${appName} - ${response.statusText}, OAuth api call failed.!`, 'error'); log(config, formatError(response), 'error'); await confirmToCloseProcess(response.data, config); @@ -132,6 +138,8 @@ export const makeRedirectUrlCall = async (response: any, appName: string, config } }) .catch((error) => { + trace(error, 'error', true); + if (includes([501, 403], error.status)) { log(config, formatError(error), 'error'); } @@ -196,5 +204,8 @@ export const updateAppConfig = async ( .then((data: any) => { log(config, `${app?.manifest?.name} app config updated successfully.!`, 'success'); }) - .catch((error: any) => log(config, `Failed to update app config.${formatError(error)}`, 'error')); + .catch((error: any) => { + trace(error, 'error', true); + log(config, `Failed to update app config.${formatError(error)}`, 'error'); + }); }; diff --git a/packages/contentstack-launch/src/commands/launch/base-command.ts b/packages/contentstack-launch/src/commands/launch/base-command.ts index 8ab9874a97..4fdb5551cb 100755 --- a/packages/contentstack-launch/src/commands/launch/base-command.ts +++ b/packages/contentstack-launch/src/commands/launch/base-command.ts @@ -11,18 +11,15 @@ import { Interfaces, cliux as ux, configHandler, + isAuthenticated, ContentstackClient, managementSDKClient, managementSDKInitiator, - isAuthenticated, } from '@contentstack/cli-utilities'; import config from '../../config'; import { GraphqlApiClient, Logger } from '../../util'; import { ConfigType, LogFn, Providers } from '../../types'; -import Functions from './functions'; -// import { logger } from '@contentstack/cli-utilities'; -import { cliux } from '@contentstack/cli-utilities'; export type Flags = Interfaces.InferredFlags<(typeof BaseCommand)['baseFlags'] & T['flags']>; export type Args = Interfaces.InferredArgs; @@ -83,8 +80,8 @@ export abstract class BaseCommand extends Command { } const _isAuthenticated = isAuthenticated(); if (!_isAuthenticated) { - cliux.print('CLI_AUTH_WHOAMI_FAILED', { color: 'yellow' }); - cliux.print('You are not logged in. Please login to execute this command, csdx auth:login', { color: 'red' }); + ux.print('CLI_AUTH_WHOAMI_FAILED', { color: 'yellow' }); + ux.print('You are not logged in. Please login to execute this command, csdx auth:login', { color: 'red' }); this.exit(1); } } @@ -156,7 +153,10 @@ export abstract class BaseCommand extends Command { choices: configKeys, message: 'Choose a branch', }) - .then((val: any) => config[val]); + .then((val: any) => config[val]) + .catch((err) => { + this.log(err, 'error'); + }); } else { this.sharedConfig.currentConfig = config[configKeys[0]]; } diff --git a/packages/contentstack-utilities/package.json b/packages/contentstack-utilities/package.json index 9df6b36e8e..39fc483641 100644 --- a/packages/contentstack-utilities/package.json +++ b/packages/contentstack-utilities/package.json @@ -44,11 +44,13 @@ "inquirer": "8.2.4", "inquirer-search-checkbox": "^1.0.0", "inquirer-search-list": "^1.2.6", - "mkdirp": "^1.0.4", + "klona": "^2.0.6", "lodash": "^4.17.15", + "mkdirp": "^1.0.4", "open": "^8.4.2", "ora": "^5.4.0", "rxjs": "^6.6.7", + "traverse": "^0.6.7", "unique-string": "^2.0.0", "uuid": "^9.0.0", "winston": "^3.7.2", @@ -62,6 +64,7 @@ "@types/mocha": "^8.2.2", "@types/node": "^14.14.32", "@types/sinon": "^10.0.2", + "@types/traverse": "^0.6.34", "chai": "^4.3.4", "eslint": "^8.18.0", "eslint-config-oclif": "^4.0.0", diff --git a/packages/contentstack-utilities/src/index.ts b/packages/contentstack-utilities/src/index.ts index ee62ee70b6..573e7570f3 100644 --- a/packages/contentstack-utilities/src/index.ts +++ b/packages/contentstack-utilities/src/index.ts @@ -1,3 +1,4 @@ +import Logger from './logger' export { LoggerService } from './logger'; export { default as cliux } from './cli-ux'; export { default as CLIError } from './cli-error'; @@ -51,3 +52,5 @@ export { export { FlagInput, ArgInput } from '@oclif/core/lib/interfaces/parser'; export { default as TablePrompt } from './inquirer-table-prompt'; + +export { Logger }; \ No newline at end of file diff --git a/packages/contentstack-utilities/src/logger.ts b/packages/contentstack-utilities/src/logger.ts index 5df0b5bad1..2120acb64f 100644 --- a/packages/contentstack-utilities/src/logger.ts +++ b/packages/contentstack-utilities/src/logger.ts @@ -1,6 +1,9 @@ -import winston from 'winston'; -import path, { resolve } from 'path'; -import messageHandler from './message-handler'; +import traverse from 'traverse'; +import { klona } from 'klona/full'; +import path, { normalize } from 'path'; +import winston, { LogEntry } from 'winston'; + +import { cliux as ux, PrintOptions, messageHandler } from './index'; export class LoggerService { name: string; @@ -15,7 +18,6 @@ export class LoggerService { this.name = null; const logger = winston.createLogger({ transports: [ - // new winston.transports.Console(), new winston.transports.File({ filename: path.resolve(process.env.CS_CLI_LOG_PATH || `${pathToLog}/logs`, `${name}.log`), }), @@ -37,16 +39,12 @@ export class LoggerService { return message; }), ), - // // level: (config.get('logger.level') as string) || 'error', - // level: 'error', - // silent: true - // silent: config.get('logger.enabled') && process.env.CLI_ENV !== 'TEST' ? false : false, }); this.logger = logger; } init(context) { - this.name = (context && context.plugin && context.plugin.name) || 'cli'; + this.name = context?.plugin?.name ?? 'cli'; } set loggerName(name: string) { @@ -96,4 +94,193 @@ export class LoggerService { this.logger.log('warn', message); } } -} \ No newline at end of file +} + +export type LogType = 'info' | 'warn' | 'error' | 'debug'; +export type LogsType = LogType | PrintOptions | undefined; +export type MessageType = string | Error | Record | Record[]; + +/* The Logger class is a TypeScript class that provides logging functionality using the winston +library, with support for redacting sensitive information and different log levels. */ +export default class Logger { + private logger: winston.Logger; + private errorLogger: winston.Logger; + private hiddenErrorLogger: winston.Logger; + private config: Record; + + /* The `sensitiveKeys` array is used to store regular expressions that match sensitive keys. These + keys are used to redact sensitive information from log messages. When logging an object, any keys + that match the regular expressions in the `sensitiveKeys` array will be replaced with the string + '[REDACTED]'. This helps to prevent sensitive information from being logged or displayed. */ + private sensitiveKeys = [ + /authtoken/i, + /^email$/, + /^password$/i, + /secret/i, + /token/i, + /api[-._]?key/i, + /management[-._]?token/i, + ]; + + /** + * The function returns an object with options for a file logger in the winston library. + * @returns an object of type `winston.transports.FileTransportOptions`. + */ + get loggerOptions(): winston.transports.FileTransportOptions { + return { + filename: '', + maxFiles: 20, + tailable: true, + maxsize: 1000000, + }; + } + + /** + * The constructor function initializes the class with a configuration object and creates logger + * instances. + * @param config - The `config` parameter is an object that contains various configuration options + * for the constructor. It is of type `Record`, which means it can have any number of + * properties of any type. + */ + constructor(config: Record) { + this.config = config; + this.logger = this.getLoggerInstance(); + this.errorLogger = this.getLoggerInstance('error'); + this.hiddenErrorLogger = this.getLoggerInstance('hidden'); + } + + /** + * The function getLoggerInstance creates and returns a winston logger instance with specified log + * level and transports. + * @param {'error' | 'info' | 'hidden'} [level=info] - The `level` parameter is an optional parameter + * that specifies the logging level. It can have one of three values: 'error', 'info', or 'hidden'. + * The default value is 'info'. + * @returns an instance of the winston.Logger class. + */ + getLoggerInstance(level: 'error' | 'info' | 'hidden' = 'info'): winston.Logger { + const filePath = normalize(process.env.CS_CLI_LOG_PATH || this.config.basePath).replace(/^(\.\.(\/|\\|$))+/, ''); + + const transports: winston.transport[] = []; + + if (level !== 'hidden') { + transports.push( + new winston.transports.Console({ + format: winston.format.combine(winston.format.colorize({ all: true })), + }), + ); + } + + transports.push( + new winston.transports.File({ + ...this.loggerOptions, + level: level === 'hidden' ? 'error' : level, + filename: `${filePath}/${level === 'hidden' ? 'error' : level}.log`, + }), + ); + + return winston.createLogger({ + levels: + level === 'error' || level === 'hidden' + ? { error: 0 } + : { + warn: 1, + info: 2, + debug: 3, + }, + level, + transports, + format: winston.format.combine( + winston.format((info) => this.redact(info))(), + winston.format.errors({ stack: level === 'hidden' }), // NOTE keep stack only for the hidden type + winston.format.simple(), + winston.format.timestamp(), + winston.format.metadata(), + ), + }); + } + + /** + * The function checks if a given key string matches any of the sensitive keys defined in an array. + * @param {string} keyStr - The parameter `keyStr` is a string that represents a key. + * @returns a boolean value. It returns true if the keyStr matches any of the regular expressions in + * the sensitiveKeys array, and false otherwise. + */ + isSensitiveKey(keyStr: string) { + if (keyStr) { + return this.sensitiveKeys.some((regex) => regex.test(keyStr)); + } + } + + /** + * The function redactObject takes an object as input and replaces any sensitive keys with the string + * '[REDACTED]'. + * @param {any} obj - The `obj` parameter is an object that you want to redact sensitive information + * from. + */ + redactObject(obj: any) { + const self = this; + traverse(obj).forEach(function redactor() { + if (self.isSensitiveKey(this.key)) { + this.update('[REDACTED]'); + } + }); + } + + /** + * The redact function takes an object, creates a copy of it, redacts sensitive information from the + * copy, and returns the redacted copy. + * @param {any} obj - The `obj` parameter is of type `any`, which means it can accept any type of + * value. It is the object that needs to be redacted. + * @returns The `redact` function is returning a copy of the `obj` parameter with certain properties + * redacted. + */ + redact(obj: any) { + const copy = klona(obj); + this.redactObject(copy); + + const splat = copy[Symbol.for('splat')]; + this.redactObject(splat); + + return copy; + } + + /** + * The function checks if an object is a LogEntry by verifying if it has the properties 'level' and + * 'message'. + * @param {any} obj - The `obj` parameter is of type `any`, which means it can be any type of value. + * @returns a boolean value. + */ + isLogEntry(obj: any): obj is LogEntry { + return typeof obj === 'object' && 'level' in obj && 'message' in obj; + } + + /* The `log` function is a method of the `Logger` class. It is used to log messages or log entries + with different log types. */ + log(entry: LogEntry): void; + log(message: MessageType, logType: LogsType): void; + log(message: MessageType, logType: 'error', hidden: boolean): void; + log(entryOrMessage: LogEntry | MessageType, logType?: LogsType, hidden?: boolean): void { + if (this.isLogEntry(entryOrMessage)) { + this.logger.log(entryOrMessage); + } else { + switch (logType) { + case 'info': + case 'debug': + case 'warn': + this.logger.log(logType, entryOrMessage); + break; + case 'error': + if (hidden) { + this.hiddenErrorLogger.error(entryOrMessage); + } else { + this.errorLogger.error(entryOrMessage); + this.hiddenErrorLogger.error(entryOrMessage); + } + break; + default: + ux.print(entryOrMessage as string, logType || {}); + break; + } + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3c621e4c0d..fc184607d0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1028,6 +1028,7 @@ importers: '@types/mocha': ^8.2.2 '@types/node': ^14.14.32 '@types/sinon': ^10.0.2 + '@types/traverse': ^0.6.34 axios: 1.3.4 chai: ^4.3.4 chalk: ^4.0.0 @@ -1044,6 +1045,7 @@ importers: inquirer: 8.2.4 inquirer-search-checkbox: ^1.0.0 inquirer-search-list: ^1.2.6 + klona: ^2.0.6 lodash: ^4.17.15 mkdirp: ^1.0.4 mocha: 10.1.0 @@ -1055,6 +1057,7 @@ importers: rxjs: ^6.6.7 sinon: ^15.0.1 tmp: ^0.2.1 + traverse: ^0.6.7 ts-node: ^10.9.1 tslib: ^1.13.0 typescript: ^4.9.3 @@ -1075,11 +1078,13 @@ importers: inquirer: 8.2.4 inquirer-search-checkbox: 1.0.0 inquirer-search-list: 1.2.6 + klona: 2.0.6 lodash: 4.17.21 mkdirp: 1.0.4 open: 8.4.2 ora: 5.4.1 rxjs: 6.6.7 + traverse: 0.6.7 unique-string: 2.0.0 uuid: 9.0.0 winston: 3.10.0 @@ -1092,6 +1097,7 @@ importers: '@types/mocha': 8.2.3 '@types/node': 14.18.53 '@types/sinon': 10.0.15 + '@types/traverse': 0.6.34 chai: 4.3.7 eslint: 8.45.0 eslint-config-oclif: 4.0.0_eslint@8.45.0 @@ -3678,6 +3684,10 @@ packages: resolution: {integrity: sha512-dDZH/tXzwjutnuk4UacGgFRwV+JSLaXL1ikvidfJprkb7L9Nx1njcRHHmi3Dsvt7pgqqTEeucQuOrWHPFgzVHA==} dev: true + /@types/traverse/0.6.34: + resolution: {integrity: sha512-bTvEo1/mx4ERGCcZfcvI8hBIUulNhqMcgyX/JzZ+p0WGdiU+oVjIJJJUs5zrkeWpfrTj98UvUm0qVHUk5lnkUg==} + dev: true + /@types/triple-beam/1.3.2: resolution: {integrity: sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==} dev: false @@ -8841,6 +8851,11 @@ packages: engines: {node: '>=6'} dev: true + /klona/2.0.6: + resolution: {integrity: sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==} + engines: {node: '>= 8'} + dev: false + /kuler/2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} dev: false @@ -12353,6 +12368,10 @@ packages: punycode: 2.3.0 dev: false + /traverse/0.6.7: + resolution: {integrity: sha512-/y956gpUo9ZNCb99YjxG7OaslxZWHfCHAUUfshwqOXmxUIvqLjVO581BT+gM59+QV9tFe6/CGG53tsA1Y7RSdg==} + dev: false + /tree-kill/1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true From 3d4dbe969fd946cd97c228e168fa8f79a5787f18 Mon Sep 17 00:00:00 2001 From: Antony Date: Mon, 23 Oct 2023 11:49:50 +0530 Subject: [PATCH 3/3] Code clean --- packages/contentstack-import/src/utils/logger.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/contentstack-import/src/utils/logger.ts b/packages/contentstack-import/src/utils/logger.ts index 2af7e031b9..139e0b4092 100644 --- a/packages/contentstack-import/src/utils/logger.ts +++ b/packages/contentstack-import/src/utils/logger.ts @@ -58,10 +58,7 @@ let errorTransport; function init(_logPath: string) { if (!logger || !errorLogger) { - let logsDir = path.resolve(_logPath, 'logs', 'import'); - // Create dir if doesn't already exist - // mkdirp.sync(logsDir); - + const logsDir = path.resolve(_logPath, 'logs', 'import'); successTransport = { filename: path.join(logsDir, 'success.log'), maxFiles: 20,