From 3625a45ef3135b4c95a32cc52f50a2d4d598aa52 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Thu, 10 Aug 2023 14:20:05 +0530 Subject: [PATCH 01/28] enable new structure --- packages/contentstack-export/src/config/index.ts | 2 +- packages/contentstack-import/src/config/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 52a0bd214b..fb12f2f075 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -402,7 +402,7 @@ const config: DefaultConfig = { writeConcurrency: 5, developerHubBaseUrl: '', marketplaceAppEncryptionKey: 'nF2ejRQcTv', - useNewModuleStructure: false, + useNewModuleStructure: true, }; export default config; diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 63fb403e5f..4a5621fd05 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -390,7 +390,7 @@ const config: DefaultConfig = { developerHubBaseUrl: '', marketplaceAppEncryptionKey: 'nF2ejRQcTv', getEncryptionKeyMaxRetry: 3, - useNewModuleStructure: false, + useNewModuleStructure: true, // useBackedupDir: '', // backupConcurrency: 10, }; From 48b541a4c4032e735fb48459adce6516ebb7e7a8 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 21 Aug 2023 15:35:10 +0530 Subject: [PATCH 02/28] fix: recursive backup directory issue --- package-lock.json | 100 ++++++++++++------ packages/contentstack-import/package.json | 6 +- .../src/utils/backup-handler.ts | 34 ++++-- pnpm-lock.yaml | 41 ++++--- 4 files changed, 117 insertions(+), 64 deletions(-) diff --git a/package-lock.json b/package-lock.json index e277ebb260..a78dc0469c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2790,6 +2790,16 @@ "integrity": "sha512-3zsplnP2djeps5P9OyarTxwRpMLoe5Ash8aL9iprw0JxB+FAHjY+ifn4yZUuW4/9hqtnmor6uvjSRzJhiVbrEQ==", "dev": true }, + "node_modules/@types/fs-extra": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.1.tgz", + "integrity": "sha512-MxObHvNl4A69ofaTRU8DFqvgzzv8s9yRtaPPm5gud9HDNvpB3GPQFvNuTWAI59B9huVGV5jXYJwbCsmBsOGYWA==", + "dev": true, + "dependencies": { + "@types/jsonfile": "*", + "@types/node": "*" + } + }, "node_modules/@types/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", @@ -2885,6 +2895,15 @@ "integrity": "sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA==", "dev": true }, + "node_modules/@types/jsonfile": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.1.tgz", + "integrity": "sha512-GSgiRCVeapDN+3pqA35IkQwasaCh/0YFH5dEF6S88iDvEn901DjOeH3/QPY+XYP1DFzDZPvIvfeEgk+7br5png==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/keyv": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz", @@ -2957,15 +2976,6 @@ "integrity": "sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==", "dev": true }, - "node_modules/@types/ncp": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@types/ncp/-/ncp-2.0.5.tgz", - "integrity": "sha512-ocK0p8JuFmX7UkMabFPjY0F7apPvQyLWt5qtdvuvQEBz9i4m2dbzV+6L1zNaUp042RfnL6pHnxDE53OH6XQ9VQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/node": { "version": "14.18.53", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.53.tgz", @@ -13361,14 +13371,6 @@ "node": "*" } }, - "node_modules/ncp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", - "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==", - "bin": { - "ncp": "bin/ncp" - } - }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -21335,11 +21337,11 @@ "bluebird": "^3.7.2", "chalk": "^4.1.2", "debug": "^4.1.0", + "fs-extra": "^11.1.1", "lodash": "^4.17.20", "marked": "^4.0.17", "merge": "^2.1.1", "mkdirp": "^1.0.4", - "ncp": "^2.0.0", "promise-limit": "^2.7.0", "tslib": "^2.4.1", "winston": "^3.7.2" @@ -21348,9 +21350,9 @@ "@oclif/test": "^1.2.6", "@types/bluebird": "^3.5.38", "@types/chai": "^4.2.18", + "@types/fs-extra": "^11.0.1", "@types/mkdirp": "^1.0.2", "@types/mocha": "^8.2.2", - "@types/ncp": "^2.0.5", "@types/node": "^14.14.32", "@types/sinon": "^10.0.2", "@types/tar": "^4.0.3", @@ -21423,6 +21425,19 @@ "@types/node": "*" } }, + "packages/contentstack-import/node_modules/fs-extra": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", + "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "packages/contentstack-import/node_modules/sinon": { "version": "11.1.2", "resolved": "https://registry.npmjs.org/sinon/-/sinon-11.1.2.tgz", @@ -24247,9 +24262,9 @@ "@oclif/test": "^1.2.6", "@types/bluebird": "^3.5.38", "@types/chai": "^4.2.18", + "@types/fs-extra": "^11.0.1", "@types/mkdirp": "^1.0.2", "@types/mocha": "^8.2.2", - "@types/ncp": "^2.0.5", "@types/node": "^14.14.32", "@types/sinon": "^10.0.2", "@types/tar": "^4.0.3", @@ -24261,13 +24276,13 @@ "debug": "^4.1.0", "eslint": "^8.18.0", "eslint-config-oclif": "^4.0.0", + "fs-extra": "^11.1.1", "globby": "^10.0.2", "lodash": "^4.17.20", "marked": "^4.0.17", "merge": "^2.1.1", "mkdirp": "^1.0.4", "mocha": "^10.0.0", - "ncp": "^2.0.0", "nyc": "^15.1.0", "oclif": "^3.8.1", "promise-limit": "^2.7.0", @@ -24328,6 +24343,16 @@ "@types/node": "*" } }, + "fs-extra": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", + "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, "sinon": { "version": "11.1.2", "resolved": "https://registry.npmjs.org/sinon/-/sinon-11.1.2.tgz", @@ -27610,6 +27635,16 @@ "integrity": "sha512-3zsplnP2djeps5P9OyarTxwRpMLoe5Ash8aL9iprw0JxB+FAHjY+ifn4yZUuW4/9hqtnmor6uvjSRzJhiVbrEQ==", "dev": true }, + "@types/fs-extra": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.1.tgz", + "integrity": "sha512-MxObHvNl4A69ofaTRU8DFqvgzzv8s9yRtaPPm5gud9HDNvpB3GPQFvNuTWAI59B9huVGV5jXYJwbCsmBsOGYWA==", + "dev": true, + "requires": { + "@types/jsonfile": "*", + "@types/node": "*" + } + }, "@types/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", @@ -27707,6 +27742,15 @@ "integrity": "sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA==", "dev": true }, + "@types/jsonfile": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.1.tgz", + "integrity": "sha512-GSgiRCVeapDN+3pqA35IkQwasaCh/0YFH5dEF6S88iDvEn901DjOeH3/QPY+XYP1DFzDZPvIvfeEgk+7br5png==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/keyv": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz", @@ -27778,15 +27822,6 @@ "integrity": "sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==", "dev": true }, - "@types/ncp": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@types/ncp/-/ncp-2.0.5.tgz", - "integrity": "sha512-ocK0p8JuFmX7UkMabFPjY0F7apPvQyLWt5qtdvuvQEBz9i4m2dbzV+6L1zNaUp042RfnL6pHnxDE53OH6XQ9VQ==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/node": { "version": "14.18.53", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.53.tgz", @@ -35681,11 +35716,6 @@ "resolved": "https://registry.npmjs.org/natural-orderby/-/natural-orderby-2.0.3.tgz", "integrity": "sha512-p7KTHxU0CUrcOXe62Zfrb5Z13nLvPhSWR/so3kFulUQU0sgUll2Z0LwpsLN351eOOD+hRGu/F1g+6xDfPeD++Q==" }, - "ncp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", - "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==" - }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index 24b6dd6e5b..a3c1c6df57 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -14,11 +14,11 @@ "bluebird": "^3.7.2", "chalk": "^4.1.2", "debug": "^4.1.0", + "fs-extra": "^11.1.1", "lodash": "^4.17.20", "marked": "^4.0.17", "merge": "^2.1.1", "mkdirp": "^1.0.4", - "ncp": "^2.0.0", "promise-limit": "^2.7.0", "tslib": "^2.4.1", "winston": "^3.7.2" @@ -27,9 +27,9 @@ "@oclif/test": "^1.2.6", "@types/bluebird": "^3.5.38", "@types/chai": "^4.2.18", + "@types/fs-extra": "^11.0.1", "@types/mkdirp": "^1.0.2", "@types/mocha": "^8.2.2", - "@types/ncp": "^2.0.5", "@types/node": "^14.14.32", "@types/sinon": "^10.0.2", "@types/tar": "^4.0.3", @@ -96,4 +96,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} \ No newline at end of file +} diff --git a/packages/contentstack-import/src/utils/backup-handler.ts b/packages/contentstack-import/src/utils/backup-handler.ts index bc87f1b5bf..e55c871a56 100755 --- a/packages/contentstack-import/src/utils/backup-handler.ts +++ b/packages/contentstack-import/src/utils/backup-handler.ts @@ -1,8 +1,10 @@ +import * as os from 'os'; import * as path from 'path'; -import ncp from 'ncp'; +import camelCase from 'lodash/camelCase'; +import { copy, copySync, removeSync } from 'fs-extra'; -import { ImportConfig } from '../types'; import { fileHelper } from './index'; +import { ImportConfig } from '../types'; export default function setupBackupDir(importConfig: ImportConfig): Promise { return new Promise(async (resolve, reject) => { @@ -20,16 +22,13 @@ export default function setupBackupDir(importConfig: ImportConfig): Promise { - if (error) { - return reject(error); - } - return resolve(backupDirPath); - }); + const currentWorkingDir = process.cwd(); + //handle error :- Cannot copy to a subdirectory of itself + if (importConfig.contentDir === currentWorkingDir || importConfig.contentDir === `${currentWorkingDir}/`) { + handleSubDirectoryError(importConfig, backupDirPath); + return resolve(backupDirPath); } else { - ncp(importConfig.contentDir, backupDirPath, (error) => { + return copy(importConfig.contentDir, backupDirPath, (error: any) => { if (error) { return reject(error); } @@ -38,3 +37,16 @@ export default function setupBackupDir(importConfig: ImportConfig): Promise { + const tempDestination = `${os.platform() === 'darwin' ? '/private/tmp' : '/tmp'}/${camelCase(backupDirPath)}`; + copySync(importConfig.contentDir, tempDestination); + copySync(tempDestination, backupDirPath); + removeSync(tempDestination); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 114cb024aa..b38ea61aa5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -665,9 +665,9 @@ importers: '@oclif/test': ^1.2.6 '@types/bluebird': ^3.5.38 '@types/chai': ^4.2.18 + '@types/fs-extra': ^11.0.1 '@types/mkdirp': ^1.0.2 '@types/mocha': ^8.2.2 - '@types/ncp': ^2.0.5 '@types/node': ^14.14.32 '@types/sinon': ^10.0.2 '@types/tar': ^4.0.3 @@ -679,13 +679,13 @@ importers: debug: ^4.1.0 eslint: ^8.18.0 eslint-config-oclif: ^4.0.0 + fs-extra: ^11.1.1 globby: ^10.0.2 lodash: ^4.17.20 marked: ^4.0.17 merge: ^2.1.1 mkdirp: ^1.0.4 mocha: ^10.0.0 - ncp: ^2.0.0 nyc: ^15.1.0 oclif: ^3.8.1 promise-limit: ^2.7.0 @@ -706,11 +706,11 @@ importers: bluebird: 3.7.2 chalk: 4.1.2 debug: 4.3.4 + fs-extra: 11.1.1 lodash: 4.17.21 marked: 4.3.0 merge: 2.1.1 mkdirp: 1.0.4 - ncp: 2.0.0 promise-limit: 2.7.0 tslib: 2.6.0 winston: 3.10.0 @@ -718,9 +718,9 @@ importers: '@oclif/test': 1.2.9 '@types/bluebird': 3.5.38 '@types/chai': 4.3.5 + '@types/fs-extra': 11.0.1 '@types/mkdirp': 1.0.2 '@types/mocha': 8.2.3 - '@types/ncp': 2.0.5 '@types/node': 14.18.53 '@types/sinon': 10.0.15 '@types/tar': 4.0.5 @@ -3222,6 +3222,13 @@ packages: resolution: {integrity: sha512-3zsplnP2djeps5P9OyarTxwRpMLoe5Ash8aL9iprw0JxB+FAHjY+ifn4yZUuW4/9hqtnmor6uvjSRzJhiVbrEQ==} dev: true + /@types/fs-extra/11.0.1: + resolution: {integrity: sha512-MxObHvNl4A69ofaTRU8DFqvgzzv8s9yRtaPPm5gud9HDNvpB3GPQFvNuTWAI59B9huVGV5jXYJwbCsmBsOGYWA==} + dependencies: + '@types/jsonfile': 6.1.1 + '@types/node': 16.18.38 + dev: true + /@types/glob/7.2.0: resolution: {integrity: sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==} dependencies: @@ -3281,6 +3288,12 @@ packages: resolution: {integrity: sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA==} dev: true + /@types/jsonfile/6.1.1: + resolution: {integrity: sha512-GSgiRCVeapDN+3pqA35IkQwasaCh/0YFH5dEF6S88iDvEn901DjOeH3/QPY+XYP1DFzDZPvIvfeEgk+7br5png==} + dependencies: + '@types/node': 16.18.38 + dev: true + /@types/keyv/3.1.4: resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} dependencies: @@ -3338,12 +3351,6 @@ packages: resolution: {integrity: sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==} dev: true - /@types/ncp/2.0.5: - resolution: {integrity: sha512-ocK0p8JuFmX7UkMabFPjY0F7apPvQyLWt5qtdvuvQEBz9i4m2dbzV+6L1zNaUp042RfnL6pHnxDE53OH6XQ9VQ==} - dependencies: - '@types/node': 16.18.38 - dev: true - /@types/node/14.18.53: resolution: {integrity: sha512-soGmOpVBUq+gaBMwom1M+krC/NNbWlosh4AtGA03SyWNDiqSKtwp7OulO1M6+mg8YkHMvJ/y0AkCeO8d1hNb7A==} @@ -6596,6 +6603,15 @@ packages: resolution: {integrity: sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==} dev: true + /fs-extra/11.1.1: + resolution: {integrity: sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==} + engines: {node: '>=14.14'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.0 + dev: false + /fs-extra/8.1.0: resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} engines: {node: '>=6 <7 || >=8'} @@ -9275,11 +9291,6 @@ packages: /natural-orderby/2.0.3: resolution: {integrity: sha512-p7KTHxU0CUrcOXe62Zfrb5Z13nLvPhSWR/so3kFulUQU0sgUll2Z0LwpsLN351eOOD+hRGu/F1g+6xDfPeD++Q==} - /ncp/2.0.0: - resolution: {integrity: sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==} - hasBin: true - dev: false - /negotiator/0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} From 5deaa262a12e8ac5740859a37ef6beaa297576a2 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 22 Aug 2023 10:48:42 +0530 Subject: [PATCH 03/28] fix: export & import logger path --- .../contentstack-export/src/commands/cm/stacks/export.ts | 1 + .../contentstack-import/src/commands/cm/stacks/import.ts | 5 ++++- packages/contentstack-import/src/utils/logger.ts | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/contentstack-export/src/commands/cm/stacks/export.ts b/packages/contentstack-export/src/commands/cm/stacks/export.ts index 02fe134816..60cef825da 100644 --- a/packages/contentstack-export/src/commands/cm/stacks/export.ts +++ b/packages/contentstack-export/src/commands/cm/stacks/export.ts @@ -111,6 +111,7 @@ export default class ExportCommand extends Command { const moduleExporter = new ModuleExporter(managementAPIClient, exportConfig); await moduleExporter.start(); log(exportConfig, `The content of the stack ${exportConfig.apiKey} has been exported successfully!`, 'success'); + log(exportConfig, `The log has been stored at '${path.join(exportDir, 'logs', 'export')}'`, 'success'); } catch (error) { log({ data: exportDir } as ExportConfig, `Failed to export stack content - ${formatError(error)}`, 'error'); log( diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index d3cc5650ed..026ece969e 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -103,22 +103,25 @@ export default class ImportCommand extends Command { // initialize the importer // start import let contentDir: string; + let backupDir: string; try { const { flags } = await this.parse(ImportCommand); let importConfig = await setupImportConfig(flags); // Note setting host to create cma client importConfig.host = this.cmaHost; contentDir = importConfig.contentDir; + backupDir = importConfig.backupDir; const managementAPIClient: ContentstackClient = await managementSDKClient(importConfig); const moduleImporter = new ModuleImporter(managementAPIClient, importConfig); await moduleImporter.start(); log(importConfig, `The content has been imported to the stack ${importConfig.apiKey} successfully!`, 'success'); + log(importConfig, `The log has been stored at '${path.join(importConfig.backupDir, 'logs', 'import')}'`, 'success'); } catch (error) { log({ data: contentDir } as ImportConfig, `Failed to import stack content - ${formatError(error)}`, 'error'); log( { data: contentDir } as ImportConfig, `The log has been stored at ${ - { data: contentDir } ? path.join(contentDir || __dirname, 'logs', 'import') : path.join(__dirname, 'logs') + { data: backupDir } ? path.join(backupDir || __dirname, 'logs', 'import') : path.join(__dirname, 'logs') }`, 'info', ); diff --git a/packages/contentstack-import/src/utils/logger.ts b/packages/contentstack-import/src/utils/logger.ts index 2b7ce63c3f..d6ca174a75 100644 --- a/packages/contentstack-import/src/utils/logger.ts +++ b/packages/contentstack-import/src/utils/logger.ts @@ -58,7 +58,7 @@ let errorTransport; function init(_logPath: string) { if (!logger || !errorLogger) { - var logsDir = path.resolve(_logPath, 'logs', 'export'); + var logsDir = path.resolve(_logPath, 'logs', 'import'); // Create dir if doesn't already exist mkdirp.sync(logsDir); From 06b66fa872e53af86e8ff1bb71dab7158b6fe835 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 23 Aug 2023 11:13:44 +0530 Subject: [PATCH 04/28] fix: entries helper --- .../contentstack-import/src/utils/entries-helper.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/contentstack-import/src/utils/entries-helper.ts b/packages/contentstack-import/src/utils/entries-helper.ts index acca676921..570be193b0 100644 --- a/packages/contentstack-import/src/utils/entries-helper.ts +++ b/packages/contentstack-import/src/utils/entries-helper.ts @@ -8,7 +8,14 @@ import config from '../config'; import * as fileHelper from './file-helper'; // update references in entry object -export const lookupEntries = function (data: any, mappedUids: Record, uidMapperPath: string) { +export const lookupEntries = function ( + data: { + content_type: any; + entry: any; + }, + mappedUids: Record, + uidMapperPath: string, +) { let parent: string[] = []; let uids: string[] = []; let unmapped: string[] = []; @@ -91,7 +98,7 @@ export const lookupEntries = function (data: any, mappedUids: Record Date: Wed, 23 Aug 2023 16:17:42 +0530 Subject: [PATCH 05/28] fixed --- packages/contentstack-auth/bin/dev | 19 +++++++++++---- packages/contentstack-bootstrap/bin/dev | 17 +++++++++++++ packages/contentstack-bootstrap/bin/dev.cmd | 3 +++ packages/contentstack-branches/bin/dev | 19 +++++++++++---- packages/contentstack-config/bin/dev | 17 +++++++++++++ packages/contentstack-config/bin/dev.cmd | 3 +++ packages/contentstack-export/bin/dev | 20 ++++++++++++---- .../src/export/modules/entries.ts | 10 ++++---- packages/contentstack-export/tsconfig.json | 2 +- .../contentstack-import/src/config/index.ts | 4 ++-- .../src/import/modules/entries.ts | 24 ++++++++++--------- .../src/import/modules/global-fields.ts | 5 ++-- .../src/utils/content-type-helper.ts | 4 +--- packages/contentstack-import/tsconfig.json | 7 +++--- packages/contentstack-seed/bin/dev | 17 +++++++++++++ packages/contentstack-seed/bin/dev.cmd | 3 +++ .../src/fs-utility/core.ts | 2 +- 17 files changed, 136 insertions(+), 40 deletions(-) create mode 100644 packages/contentstack-bootstrap/bin/dev create mode 100644 packages/contentstack-bootstrap/bin/dev.cmd create mode 100644 packages/contentstack-config/bin/dev create mode 100644 packages/contentstack-config/bin/dev.cmd create mode 100644 packages/contentstack-seed/bin/dev create mode 100644 packages/contentstack-seed/bin/dev.cmd diff --git a/packages/contentstack-auth/bin/dev b/packages/contentstack-auth/bin/dev index f003f9d180..02b50d62ab 100644 --- a/packages/contentstack-auth/bin/dev +++ b/packages/contentstack-auth/bin/dev @@ -1,6 +1,17 @@ #!/usr/bin/env node -(async () => { - const { execute } = require('@contentstack/cli-utilities'); - await execute({ type: 'cjs', development: true, dir: __dirname }); -})(); \ No newline at end of file +const oclif = require('@oclif/core'); + +const path = require('path'); +const project = path.join(__dirname, '..', 'tsconfig.json'); + +// In dev mode -> use ts-node and dev plugins +process.env.NODE_ENV = 'development'; + +require('ts-node').register({ project }); + +// In dev mode, always show stack traces +oclif.settings.debug = true; + +// Start the CLI +oclif.run().then(oclif.flush).catch(oclif.Errors.handle); diff --git a/packages/contentstack-bootstrap/bin/dev b/packages/contentstack-bootstrap/bin/dev new file mode 100644 index 0000000000..02b50d62ab --- /dev/null +++ b/packages/contentstack-bootstrap/bin/dev @@ -0,0 +1,17 @@ +#!/usr/bin/env node + +const oclif = require('@oclif/core'); + +const path = require('path'); +const project = path.join(__dirname, '..', 'tsconfig.json'); + +// In dev mode -> use ts-node and dev plugins +process.env.NODE_ENV = 'development'; + +require('ts-node').register({ project }); + +// In dev mode, always show stack traces +oclif.settings.debug = true; + +// Start the CLI +oclif.run().then(oclif.flush).catch(oclif.Errors.handle); diff --git a/packages/contentstack-bootstrap/bin/dev.cmd b/packages/contentstack-bootstrap/bin/dev.cmd new file mode 100644 index 0000000000..8ae2b12c19 --- /dev/null +++ b/packages/contentstack-bootstrap/bin/dev.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\dev" %* diff --git a/packages/contentstack-branches/bin/dev b/packages/contentstack-branches/bin/dev index 59b22a63af..02b50d62ab 100755 --- a/packages/contentstack-branches/bin/dev +++ b/packages/contentstack-branches/bin/dev @@ -1,6 +1,17 @@ #!/usr/bin/env node -(async () => { - const oclif = require('@oclif/core'); - await oclif.execute({ type: 'cjs', development: true, dir: __dirname }); -})(); +const oclif = require('@oclif/core'); + +const path = require('path'); +const project = path.join(__dirname, '..', 'tsconfig.json'); + +// In dev mode -> use ts-node and dev plugins +process.env.NODE_ENV = 'development'; + +require('ts-node').register({ project }); + +// In dev mode, always show stack traces +oclif.settings.debug = true; + +// Start the CLI +oclif.run().then(oclif.flush).catch(oclif.Errors.handle); diff --git a/packages/contentstack-config/bin/dev b/packages/contentstack-config/bin/dev new file mode 100644 index 0000000000..02b50d62ab --- /dev/null +++ b/packages/contentstack-config/bin/dev @@ -0,0 +1,17 @@ +#!/usr/bin/env node + +const oclif = require('@oclif/core'); + +const path = require('path'); +const project = path.join(__dirname, '..', 'tsconfig.json'); + +// In dev mode -> use ts-node and dev plugins +process.env.NODE_ENV = 'development'; + +require('ts-node').register({ project }); + +// In dev mode, always show stack traces +oclif.settings.debug = true; + +// Start the CLI +oclif.run().then(oclif.flush).catch(oclif.Errors.handle); diff --git a/packages/contentstack-config/bin/dev.cmd b/packages/contentstack-config/bin/dev.cmd new file mode 100644 index 0000000000..8ae2b12c19 --- /dev/null +++ b/packages/contentstack-config/bin/dev.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\dev" %* diff --git a/packages/contentstack-export/bin/dev b/packages/contentstack-export/bin/dev index 74555287cd..02b50d62ab 100755 --- a/packages/contentstack-export/bin/dev +++ b/packages/contentstack-export/bin/dev @@ -1,5 +1,17 @@ #!/usr/bin/env node -(async () => { - const oclif = require('@oclif/core'); - await oclif.execute({ type: 'cjs', development: true, dir: __dirname }); -})(); + +const oclif = require('@oclif/core'); + +const path = require('path'); +const project = path.join(__dirname, '..', 'tsconfig.json'); + +// In dev mode -> use ts-node and dev plugins +process.env.NODE_ENV = 'development'; + +require('ts-node').register({ project }); + +// In dev mode, always show stack traces +oclif.settings.debug = true; + +// Start the CLI +oclif.run().then(oclif.flush).catch(oclif.Errors.handle); diff --git a/packages/contentstack-export/src/export/modules/entries.ts b/packages/contentstack-export/src/export/modules/entries.ts index f795cee78c..b1981929cf 100644 --- a/packages/contentstack-export/src/export/modules/entries.ts +++ b/packages/contentstack-export/src/export/modules/entries.ts @@ -54,11 +54,11 @@ export default class EntriesExport extends BaseClass { } const entryRequestOptions = this.createRequestObjects(locales, contentTypes); for (let entryRequestOption of entryRequestOptions) { - log( - this.exportConfig, - `Starting export of entries of content type - ${entryRequestOption.contentType} locale - ${entryRequestOption.locale}`, - 'info', - ); + // log( + // this.exportConfig, + // `Starting export of entries of content type - ${entryRequestOption.contentType} locale - ${entryRequestOption.locale}`, + // 'info', + // ); await this.getEntries(entryRequestOption); this.entriesFileHelper?.completeFile(true); log( diff --git a/packages/contentstack-export/tsconfig.json b/packages/contentstack-export/tsconfig.json index 5136623392..da8fc4b724 100644 --- a/packages/contentstack-export/tsconfig.json +++ b/packages/contentstack-export/tsconfig.json @@ -12,7 +12,7 @@ "skipLibCheck": true, "sourceMap": false, "esModuleInterop": true, - "noImplicitAny": true, + "noImplicitAny": false, "lib": [ "ES2019", "es2020.promise" diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 4a5621fd05..d1745a1288 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -84,9 +84,9 @@ const config: DefaultConfig = { publishAssets: true, fileName: 'assets.json', importSameStructure: true, - uploadAssetsConcurrency: 10, + uploadAssetsConcurrency: 5, displayExecutionTime: false, - importFoldersConcurrency: 10, + importFoldersConcurrency: 1, includeVersionedAssets: false, host: 'https://api.contentstack.io', folderValidKeys: ['name', 'parent_uid'], diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index cf84edfe08..33dbb83f3d 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -209,14 +209,6 @@ export default class EntriesImport extends BaseClass { jsonRteEmbeddedEntries: false, }; suppressSchemaReference(contentType.schema, flag); - // Check if suppress modified flag - if (flag.suppressed) { - this.modifiedCTs.push(find(this.cTs, { uid: contentType.uid })); - } else { - // Note: Skips the content type from update if no reference found - apiOptions.additionalInfo = { skip: true }; - return apiOptions; - } if (flag.references) { this.refCTs.push(contentType.uid); @@ -231,6 +223,16 @@ export default class EntriesImport extends BaseClass { } } } + + // Check if suppress modified flag + if (flag.suppressed) { + this.modifiedCTs.push(find(this.cTs, { uid: contentType.uid })); + } else { + // Note: Skips the content type from update if no reference found + apiOptions.additionalInfo = { skip: true }; + return apiOptions; + } + lookupExtension( this.importConfig, contentType.schema, @@ -266,7 +268,7 @@ export default class EntriesImport extends BaseClass { if (indexerCount === 0) { return Promise.resolve(); } - log(this.importConfig, `Starting to create entries for ${cTUid} in locale ${locale}`, 'info'); + // log(this.importConfig, `Starting to create entries for ${cTUid} in locale ${locale}`, 'info'); const isMasterLocale = locale === this.importConfig?.master_locale?.code; // Write created entries const entriesCreateFileHelper = new FsUtility({ @@ -381,7 +383,7 @@ export default class EntriesImport extends BaseClass { if (indexerCount === 0) { return Promise.resolve(); } - log(this.importConfig, `Starting to update entries with references for ${cTUid} in locale ${locale}`, 'info'); + // log(this.importConfig, `Starting to update entries with references for ${cTUid} in locale ${locale}`, 'info'); const contentType = find(this.cTs, { uid: cTUid }); @@ -590,7 +592,7 @@ export default class EntriesImport extends BaseClass { if (indexerCount === 0) { return Promise.resolve(); } - log(this.importConfig, `Starting publish entries for ${cTUid} in locale ${locale}`, 'info'); + // log(this.importConfig, `Starting publish entries for ${cTUid} in locale ${locale}`, 'info'); const onSuccess = ({ response, apiData: { environments }, additionalInfo: { entryUid } }: any) => { log( diff --git a/packages/contentstack-import/src/import/modules/global-fields.ts b/packages/contentstack-import/src/import/modules/global-fields.ts index c0fe880c30..958ada0c17 100644 --- a/packages/contentstack-import/src/import/modules/global-fields.ts +++ b/packages/contentstack-import/src/import/modules/global-fields.ts @@ -116,8 +116,9 @@ export default class ImportGlobalFields extends BaseClass { }) { return new Promise(async (resolve, reject) => { lookupExtension(this.config, globalField.schema, this.config.preserveStackVersion, this.installedExtensions); - const isReferenceFieldRemoved = await removeReferenceFields(globalField.schema, undefined, this.stackAPIClient); - if (isReferenceFieldRemoved) { + let flag = { supressed: false }; + await removeReferenceFields(globalField.schema, flag, this.stackAPIClient); + if (flag.supressed) { this.pendingGFs.push(globalField.uid); } return this.stack diff --git a/packages/contentstack-import/src/utils/content-type-helper.ts b/packages/contentstack-import/src/utils/content-type-helper.ts index 72c2778143..3644cbc750 100644 --- a/packages/contentstack-import/src/utils/content-type-helper.ts +++ b/packages/contentstack-import/src/utils/content-type-helper.ts @@ -90,7 +90,7 @@ export const removeReferenceFields = async function ( for (var block in schema[i].blocks) { await removeReferenceFields(schema[i].blocks[block].schema, flag, stackAPIClient); } - } else if (schema[i].data_type === 'reference' || schema[i].reference_to) { + } else if (schema[i].data_type === 'reference') { flag.supressed = true; // Check if content-type exists // If exists, then no change should be required. @@ -128,7 +128,6 @@ export const removeReferenceFields = async function ( }); } } - return true; } else if ( // handling entry references in json rte schema[i].data_type === 'json' && @@ -138,7 +137,6 @@ export const removeReferenceFields = async function ( ) { flag.supressed = true; schema[i].reference_to = ['sys_assets']; - return true; } } }; diff --git a/packages/contentstack-import/tsconfig.json b/packages/contentstack-import/tsconfig.json index 679afbeaf5..468adb221f 100644 --- a/packages/contentstack-import/tsconfig.json +++ b/packages/contentstack-import/tsconfig.json @@ -4,6 +4,7 @@ "declaration": true, "importHelpers": true, "module": "commonjs", + "rootDir": "src", "outDir": "lib", "strict": false, "target": "es2017", @@ -11,10 +12,10 @@ "skipLibCheck": true, "sourceMap": false, "esModuleInterop": true, - "noImplicitAny": true, + "noImplicitAny": false, "lib": [ - "es2019", - "es2020.promise", + "ES2019", + "es2020.promise" ], "strictPropertyInitialization": false, "forceConsistentCasingInFileNames": true diff --git a/packages/contentstack-seed/bin/dev b/packages/contentstack-seed/bin/dev new file mode 100644 index 0000000000..02b50d62ab --- /dev/null +++ b/packages/contentstack-seed/bin/dev @@ -0,0 +1,17 @@ +#!/usr/bin/env node + +const oclif = require('@oclif/core'); + +const path = require('path'); +const project = path.join(__dirname, '..', 'tsconfig.json'); + +// In dev mode -> use ts-node and dev plugins +process.env.NODE_ENV = 'development'; + +require('ts-node').register({ project }); + +// In dev mode, always show stack traces +oclif.settings.debug = true; + +// Start the CLI +oclif.run().then(oclif.flush).catch(oclif.Errors.handle); diff --git a/packages/contentstack-seed/bin/dev.cmd b/packages/contentstack-seed/bin/dev.cmd new file mode 100644 index 0000000000..8ae2b12c19 --- /dev/null +++ b/packages/contentstack-seed/bin/dev.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\dev" %* diff --git a/packages/contentstack-utilities/src/fs-utility/core.ts b/packages/contentstack-utilities/src/fs-utility/core.ts index 13d796d77d..dcfaea4ad8 100644 --- a/packages/contentstack-utilities/src/fs-utility/core.ts +++ b/packages/contentstack-utilities/src/fs-utility/core.ts @@ -285,8 +285,8 @@ export default class FsUtility { if (this.fileExt === 'json') { this.writableStream.write('}'); } - this.closeFile(closeIndexer); } + this.closeFile(closeIndexer); } /** From 3d1629e92bce21bc8b42ef1023758a2b35aac22d Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 24 Aug 2023 15:20:39 +0530 Subject: [PATCH 06/28] fix: handle subdirectory error --- .../src/import/module-importer.ts | 1 + .../src/utils/backup-handler.ts | 53 +++++++++---------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index 1ce2c624ca..df808bfbe0 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -48,6 +48,7 @@ class ModuleImporter { } const backupDir = await backupHandler(this.importConfig); if (backupDir) { + log(this.importConfig, `Backup directory path - '${backupDir}'`, 'info'); this.importConfig.backupDir = backupDir; // To support the old config this.importConfig.data = backupDir; diff --git a/packages/contentstack-import/src/utils/backup-handler.ts b/packages/contentstack-import/src/utils/backup-handler.ts index e55c871a56..cd2c28fa21 100755 --- a/packages/contentstack-import/src/utils/backup-handler.ts +++ b/packages/contentstack-import/src/utils/backup-handler.ts @@ -1,7 +1,5 @@ -import * as os from 'os'; import * as path from 'path'; -import camelCase from 'lodash/camelCase'; -import { copy, copySync, removeSync } from 'fs-extra'; +import { copy } from 'fs-extra'; import { fileHelper } from './index'; import { ImportConfig } from '../types'; @@ -12,22 +10,25 @@ export default function setupBackupDir(importConfig: ImportConfig): Promise { if (error) { return reject(error); @@ -38,15 +39,11 @@ export default function setupBackupDir(importConfig: ImportConfig): Promise { - const tempDestination = `${os.platform() === 'darwin' ? '/private/tmp' : '/tmp'}/${camelCase(backupDirPath)}`; - copySync(importConfig.contentDir, tempDestination); - copySync(tempDestination, backupDirPath); - removeSync(tempDestination); -}; +function isSubDirectory(parent: string, child: string) { + const relative = path.relative(parent, child); + if (relative) { + return relative && !relative.startsWith('..') && !path.isAbsolute(relative); + } else { + return false; + } +} From 66baec21a9d557c1ddc71f0e351453c9f72ce2a9 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Thu, 24 Aug 2023 19:50:42 +0530 Subject: [PATCH 07/28] entries fixes --- .../contentstack-export/src/config/index.ts | 4 +- .../src/import/modules/assets.ts | 7 +- .../src/import/modules/base-class.ts | 13 ++-- .../src/import/modules/entries.ts | 65 ++++++++++++------- 4 files changed, 57 insertions(+), 32 deletions(-) diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index fb12f2f075..9a98fdcc8a 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -96,8 +96,8 @@ const config: DefaultConfig = { invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by', '_metadata', 'published'], // no of asset version files (of a single asset) that'll be downloaded parallel chunkFileSize: 1, // measured on Megabits (5mb) - downloadLimit: 10, - fetchConcurrency: 10, + downloadLimit: 5, + fetchConcurrency: 5, assetsMetaKeys: [], // Default keys ['uid', 'url', 'filename'] securedAssets: false, displayExecutionTime: false, diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index 1863bed5a8..8b048da5bf 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -4,6 +4,7 @@ import filter from 'lodash/filter'; import unionBy from 'lodash/unionBy'; import orderBy from 'lodash/orderBy'; import isEmpty from 'lodash/isEmpty'; +import uniq from 'lodash/uniq'; import { existsSync } from 'node:fs'; import includes from 'lodash/includes'; import { resolve as pResolve, join } from 'node:path'; @@ -258,9 +259,11 @@ export default class ImportAssets extends BaseClass { }; const serializeData = (apiOptions: ApiOptions) => { const { apiData: asset } = apiOptions; - const publishDetails = filter(asset.publish_details, 'environment'); + const publishDetails = filter(asset.publish_details, ({ environment }) => { + return this.environments.hasOwnProperty(environment); + }); + const environments = uniq(map(publishDetails, ({ environment }) => this.environments[environment].name)); const locales = map(publishDetails, 'locale'); - const environments = map(publishDetails, ({ environment }) => this.environments[environment].name); asset.locales = locales; asset.environments = environments; diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 932e375235..5ec7424afd 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -299,8 +299,8 @@ export default abstract class BaseClass { case 'create-cts': return this.stack.contentType().create(apiData).then(onSuccess).catch(onReject); case 'update-cts': - if (additionalInfo.skip) { - return Promise.resolve(onSuccess(apiData)); + if (!apiData) { + return Promise.resolve(); } return apiData.update().then(onSuccess).catch(onReject); case 'update-gfs': @@ -345,6 +345,9 @@ export default abstract class BaseClass { .then(onSuccess) .catch(onReject); case 'create-entries': + if (additionalInfo[apiData?.uid]?.isLocalized) { + return apiData.update({ locale: additionalInfo.locale }).then(onSuccess).catch(onReject); + } return this.stack .contentType(additionalInfo.cTUid) .entry() @@ -354,8 +357,8 @@ export default abstract class BaseClass { case 'update-entries': return apiData.update({ locale: additionalInfo.locale }).then(onSuccess).catch(onReject); case 'publish-entries': - if (additionalInfo.skip) { - return Promise.resolve(onSuccess(apiData)); + if (!apiData) { + return Promise.resolve(); } return this.stack .contentType(additionalInfo.cTUid) @@ -367,7 +370,7 @@ export default abstract class BaseClass { return this.stack .contentType(apiData.cTUid) .entry(apiData.entryUid) - .delete({ locale: this.importConfig?.master_locale?.code }) + .delete({ locale: additionalInfo.locale }) .then(onSuccess) .catch(onReject); default: diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 33dbb83f3d..71b7684f09 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -80,6 +80,7 @@ export default class EntriesImport extends BaseClass { this.jsonRteCTsWithRef = []; this.envs = {}; this.autoCreatedEntries = []; + this.failedEntries = []; } async start(): Promise { @@ -109,6 +110,17 @@ export default class EntriesImport extends BaseClass { await fileHelper.writeLargeFile(path.join(this.entriesMapperPath, 'uid-mapping.json'), this.entriesUidMapper); // TBD: manages mapper in one file, should find an alternative fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); + if (this.autoCreatedEntries.length > 0) { + log(this.importConfig, 'Removing entries from master language which got created by default', 'info'); + await this.removeAutoCreatedEntries().catch((error) => { + log( + this.importConfig, + `Error while removing auto created entries in master locale ${formatError(error)}`, + 'error', + ); + }); + } + // Update entries with references const entryUpdateRequestOptions = this.populateEntryUpdatePayload(); for (let entryUpdateRequestOption of entryUpdateRequestOptions) { @@ -128,16 +140,6 @@ export default class EntriesImport extends BaseClass { log(this.importConfig, `Error while updating content type references ${formatError(error)}`, 'error'); }); - if (this.autoCreatedEntries.length > 0) { - log(this.importConfig, 'Removing entries from master language which got created by default', 'info'); - await this.removeAutoCreatedEntries().catch((error) => { - log( - this.importConfig, - `Error while removing auto created entries in master locale ${formatError(error)}`, - 'error', - ); - }); - } // Update field rule of content types which are got removed earlier log(this.importConfig, 'Updating the field rules of content type', 'info'); await this.updateFieldRules().catch((error) => { @@ -229,7 +231,7 @@ export default class EntriesImport extends BaseClass { this.modifiedCTs.push(find(this.cTs, { uid: contentType.uid })); } else { // Note: Skips the content type from update if no reference found - apiOptions.additionalInfo = { skip: true }; + apiOptions.apiData = null; return apiOptions; } @@ -272,7 +274,7 @@ export default class EntriesImport extends BaseClass { const isMasterLocale = locale === this.importConfig?.master_locale?.code; // Write created entries const entriesCreateFileHelper = new FsUtility({ - moduleName: 'created-entries', + moduleName: 'entries', indexFileName: 'index.json', basePath: path.join(this.entriesMapperPath, cTUid, locale), chunkFileSize: this.entriesConfig.chunkFileSize, @@ -281,15 +283,15 @@ export default class EntriesImport extends BaseClass { }); const contentType = find(this.cTs, { uid: cTUid }); - const onSuccess = ({ response, apiData: entry, additionalInfo: { entryFileName } }: any) => { + const onSuccess = ({ response, apiData: entry, additionalInfo }: any) => { log(this.importConfig, `Created entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info'); - this.entriesUidMapper[entry.uid] = response.uid; - entry.sourceEntryFilePath = path.join(basePath, entryFileName); // stores source file path temporarily - entry.entryOldUid = entry.uid; // stores old uid temporarily - if (!isMasterLocale) { + if (!isMasterLocale && !additionalInfo[entry.uid]?.isLocalized) { this.autoCreatedEntries.push({ cTUid, locale, entryUid: response.uid }); } - entriesCreateFileHelper.writeIntoFile({ [response.uid]: entry } as any, { mapKeyVal: true }); + this.entriesUidMapper[entry.uid] = response.uid; + entry.sourceEntryFilePath = path.join(basePath, additionalInfo.entryFileName); // stores source file path temporarily + entry.entryOldUid = entry.uid; // stores old uid temporarily + entriesCreateFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true }); }; const onReject = ({ error, apiData: { uid, title } }: any) => { log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to create`, 'error'); @@ -315,7 +317,7 @@ export default class EntriesImport extends BaseClass { entity: 'create-entries', includeParamOnCompletion: true, serializeData: this.serializeEntries.bind(this), - additionalInfo: { contentType, locale, cTUid, entryFileName: indexer[index] }, + additionalInfo: { contentType, locale, cTUid, entryFileName: indexer[index], isMasterLocale }, }, concurrencyLimit: this.importConcurrency, }).then(() => { @@ -334,7 +336,7 @@ export default class EntriesImport extends BaseClass { serializeEntries(apiOptions: ApiOptions): ApiOptions { let { apiData: entry, - additionalInfo: { cTUid, locale, contentType }, + additionalInfo: { cTUid, locale, contentType, isMasterLocale }, } = apiOptions; if (this.jsonRteCTs.indexOf(cTUid) > -1) { @@ -356,6 +358,16 @@ export default class EntriesImport extends BaseClass { this.installedExtensions, ); delete entry.publish_details; + // checking the entry is a localized one or not + if (!isMasterLocale && this.entriesUidMapper.hasOwnProperty(entry.uid)) { + const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry.uid]); + Object.assign(entryResponse, cloneDeep(entry), { uid: this.entriesUidMapper[entry.uid] }); + apiOptions.apiData = entryResponse; + apiOptions.additionalInfo[entryResponse.uid] = { + isLocalized: true, + }; + return apiOptions; + } apiOptions.apiData = entry; return apiOptions; } @@ -393,7 +405,12 @@ export default class EntriesImport extends BaseClass { const onReject = ({ error, apiData: { uid, title } }: any) => { log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to update`, 'error'); log(this.importConfig, formatError(error), 'error'); - this.failedEntries.push({ content_type: cTUid, locale, entry: { uid: this.entriesUidMapper[uid], title } }); + this.failedEntries.push({ + content_type: cTUid, + locale, + entry: { uid: this.entriesUidMapper[uid], title }, + entryId: uid, + }); }; for (const index in indexer) { @@ -458,7 +475,7 @@ export default class EntriesImport extends BaseClass { ); const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry.uid]); - Object.assign(entryResponse, cloneDeep(entry)); + Object.assign(entryResponse, cloneDeep(entry), { uid: this.entriesUidMapper[entry.uid] }); delete entryResponse.publish_details; apiOptions.apiData = entryResponse; return apiOptions; @@ -527,6 +544,7 @@ export default class EntriesImport extends BaseClass { resolve: onSuccess.bind(this), entity: 'delete-entries', includeParamOnCompletion: true, + additionalInfo: { locale: this.importConfig?.master_locale?.code }, }, concurrencyLimit: this.importConcurrency, }); @@ -668,7 +686,8 @@ export default class EntriesImport extends BaseClass { } }); } else { - additionalInfo.skip = true; + additionalInfo.apiData = null; + return apiOptions; } apiOptions.apiData = requestObject; return apiOptions; From bb1ba625f4beaef1d409ff853a945042cccf238c Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Fri, 25 Aug 2023 15:43:13 +0530 Subject: [PATCH 08/28] refactor: backup directory --- .../src/utils/backup-handler.ts | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/contentstack-import/src/utils/backup-handler.ts b/packages/contentstack-import/src/utils/backup-handler.ts index cd2c28fa21..802a727896 100755 --- a/packages/contentstack-import/src/utils/backup-handler.ts +++ b/packages/contentstack-import/src/utils/backup-handler.ts @@ -1,5 +1,6 @@ import * as path from 'path'; import { copy } from 'fs-extra'; +import { cliux } from '@contentstack/cli-utilities'; import { fileHelper } from './index'; import { ImportConfig } from '../types'; @@ -10,11 +11,17 @@ export default function setupBackupDir(importConfig: ImportConfig): Promise Date: Fri, 25 Aug 2023 18:13:03 +0530 Subject: [PATCH 09/28] publish fixes --- packages/contentstack-auth/bin/dev | 19 ++++--------------- packages/contentstack-bootstrap/bin/dev | 19 ++++--------------- packages/contentstack-branches/bin/dev | 19 ++++--------------- packages/contentstack-config/bin/dev | 19 ++++--------------- packages/contentstack-export/bin/dev | 19 ++++--------------- packages/contentstack-import/bin/dev | 19 ++++--------------- .../src/import/modules/base-class.ts | 7 +++++-- .../src/import/modules/entries.ts | 11 ++++++----- packages/contentstack-launch/bin/dev | 19 ++++--------------- packages/contentstack-seed/bin/dev | 19 ++++--------------- 10 files changed, 43 insertions(+), 127 deletions(-) diff --git a/packages/contentstack-auth/bin/dev b/packages/contentstack-auth/bin/dev index 02b50d62ab..39382a0159 100644 --- a/packages/contentstack-auth/bin/dev +++ b/packages/contentstack-auth/bin/dev @@ -1,17 +1,6 @@ #!/usr/bin/env node -const oclif = require('@oclif/core'); - -const path = require('path'); -const project = path.join(__dirname, '..', 'tsconfig.json'); - -// In dev mode -> use ts-node and dev plugins -process.env.NODE_ENV = 'development'; - -require('ts-node').register({ project }); - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle); +(async () => { + const { execute } = require('@contentstack/cli-utilities'); + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); diff --git a/packages/contentstack-bootstrap/bin/dev b/packages/contentstack-bootstrap/bin/dev index 02b50d62ab..39382a0159 100644 --- a/packages/contentstack-bootstrap/bin/dev +++ b/packages/contentstack-bootstrap/bin/dev @@ -1,17 +1,6 @@ #!/usr/bin/env node -const oclif = require('@oclif/core'); - -const path = require('path'); -const project = path.join(__dirname, '..', 'tsconfig.json'); - -// In dev mode -> use ts-node and dev plugins -process.env.NODE_ENV = 'development'; - -require('ts-node').register({ project }); - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle); +(async () => { + const { execute } = require('@contentstack/cli-utilities'); + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); diff --git a/packages/contentstack-branches/bin/dev b/packages/contentstack-branches/bin/dev index 02b50d62ab..59b22a63af 100755 --- a/packages/contentstack-branches/bin/dev +++ b/packages/contentstack-branches/bin/dev @@ -1,17 +1,6 @@ #!/usr/bin/env node -const oclif = require('@oclif/core'); - -const path = require('path'); -const project = path.join(__dirname, '..', 'tsconfig.json'); - -// In dev mode -> use ts-node and dev plugins -process.env.NODE_ENV = 'development'; - -require('ts-node').register({ project }); - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle); +(async () => { + const oclif = require('@oclif/core'); + await oclif.execute({ type: 'cjs', development: true, dir: __dirname }); +})(); diff --git a/packages/contentstack-config/bin/dev b/packages/contentstack-config/bin/dev index 02b50d62ab..39382a0159 100644 --- a/packages/contentstack-config/bin/dev +++ b/packages/contentstack-config/bin/dev @@ -1,17 +1,6 @@ #!/usr/bin/env node -const oclif = require('@oclif/core'); - -const path = require('path'); -const project = path.join(__dirname, '..', 'tsconfig.json'); - -// In dev mode -> use ts-node and dev plugins -process.env.NODE_ENV = 'development'; - -require('ts-node').register({ project }); - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle); +(async () => { + const { execute } = require('@contentstack/cli-utilities'); + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); diff --git a/packages/contentstack-export/bin/dev b/packages/contentstack-export/bin/dev index 02b50d62ab..59b22a63af 100755 --- a/packages/contentstack-export/bin/dev +++ b/packages/contentstack-export/bin/dev @@ -1,17 +1,6 @@ #!/usr/bin/env node -const oclif = require('@oclif/core'); - -const path = require('path'); -const project = path.join(__dirname, '..', 'tsconfig.json'); - -// In dev mode -> use ts-node and dev plugins -process.env.NODE_ENV = 'development'; - -require('ts-node').register({ project }); - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle); +(async () => { + const oclif = require('@oclif/core'); + await oclif.execute({ type: 'cjs', development: true, dir: __dirname }); +})(); diff --git a/packages/contentstack-import/bin/dev b/packages/contentstack-import/bin/dev index 02b50d62ab..59b22a63af 100644 --- a/packages/contentstack-import/bin/dev +++ b/packages/contentstack-import/bin/dev @@ -1,17 +1,6 @@ #!/usr/bin/env node -const oclif = require('@oclif/core'); - -const path = require('path'); -const project = path.join(__dirname, '..', 'tsconfig.json'); - -// In dev mode -> use ts-node and dev plugins -process.env.NODE_ENV = 'development'; - -require('ts-node').register({ project }); - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle); +(async () => { + const oclif = require('@oclif/core'); + await oclif.execute({ type: 'cjs', development: true, dir: __dirname }); +})(); diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 5ec7424afd..3510d8ae88 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -362,8 +362,11 @@ export default abstract class BaseClass { } return this.stack .contentType(additionalInfo.cTUid) - .entry(additionalInfo.entryUid) - .publish({ publishDetails: apiData, locale: additionalInfo.locale }) + .entry(apiData.entryUid) + .publish({ + publishDetails: { environments: apiData.environments, locales: apiData.locales }, + locale: additionalInfo.locale, + }) .then(onSuccess) .catch(onReject); case 'delete-entries': diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 71b7684f09..1188d18f90 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -612,7 +612,7 @@ export default class EntriesImport extends BaseClass { } // log(this.importConfig, `Starting publish entries for ${cTUid} in locale ${locale}`, 'info'); - const onSuccess = ({ response, apiData: { environments }, additionalInfo: { entryUid } }: any) => { + const onSuccess = ({ response, apiData: { environments, entryUid }, additionalInfo }: any) => { log( this.importConfig, `Published entry: '${entryUid}' of content type ${cTUid} and locale ${locale} in ${environments?.join( @@ -621,10 +621,10 @@ export default class EntriesImport extends BaseClass { 'info', ); }; - const onReject = ({ error, apiData, additionalInfo: { entryUid } }: any) => { + const onReject = ({ error, apiData, additionalInfo }: any) => { log( this.importConfig, - `${entryUid} entry of content type ${cTUid} in locale ${locale} failed to publish`, + `${apiData.entryUid} entry of content type ${cTUid} in locale ${locale} failed to publish`, 'error', ); log(this.importConfig, formatError(error), 'error'); @@ -665,13 +665,14 @@ export default class EntriesImport extends BaseClass { */ serializePublishEntries(apiOptions: ApiOptions): ApiOptions { let { apiData: entry, additionalInfo } = apiOptions; - additionalInfo.entryUid = this.entriesUidMapper[entry.uid]; const requestObject: { environments: Array; locales: Array; + entryUid: string; } = { environments: [], locales: [], + entryUid: this.entriesUidMapper[entry.uid], }; if (entry.publish_details && entry.publish_details.length > 0) { forEach(entry.publish_details, (pubObject) => { @@ -686,7 +687,7 @@ export default class EntriesImport extends BaseClass { } }); } else { - additionalInfo.apiData = null; + apiOptions.apiData = null; return apiOptions; } apiOptions.apiData = requestObject; diff --git a/packages/contentstack-launch/bin/dev b/packages/contentstack-launch/bin/dev index 794d701677..39382a0159 100755 --- a/packages/contentstack-launch/bin/dev +++ b/packages/contentstack-launch/bin/dev @@ -1,17 +1,6 @@ #!/usr/bin/env node -require("dotenv").config(); -const oclif = require('@oclif/core') -const path = require('path') -const project = path.join(__dirname, '..', 'tsconfig.json') - -// In dev mode -> use ts-node and dev plugins -process.env.NODE_ENV = 'development' - -require('ts-node').register({project }) - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle) +(async () => { + const { execute } = require('@contentstack/cli-utilities'); + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); diff --git a/packages/contentstack-seed/bin/dev b/packages/contentstack-seed/bin/dev index 02b50d62ab..39382a0159 100644 --- a/packages/contentstack-seed/bin/dev +++ b/packages/contentstack-seed/bin/dev @@ -1,17 +1,6 @@ #!/usr/bin/env node -const oclif = require('@oclif/core'); - -const path = require('path'); -const project = path.join(__dirname, '..', 'tsconfig.json'); - -// In dev mode -> use ts-node and dev plugins -process.env.NODE_ENV = 'development'; - -require('ts-node').register({ project }); - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle); +(async () => { + const { execute } = require('@contentstack/cli-utilities'); + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); From b12df65ce248b552b7a5848de8a92a40430f75f5 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Mon, 28 Aug 2023 10:42:24 +0530 Subject: [PATCH 10/28] backup warning message update --- packages/contentstack-import/src/utils/backup-handler.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/contentstack-import/src/utils/backup-handler.ts b/packages/contentstack-import/src/utils/backup-handler.ts index 802a727896..63fb50b84f 100755 --- a/packages/contentstack-import/src/utils/backup-handler.ts +++ b/packages/contentstack-import/src/utils/backup-handler.ts @@ -17,9 +17,12 @@ export default function setupBackupDir(importConfig: ImportConfig): Promise Date: Mon, 28 Aug 2023 10:50:28 +0530 Subject: [PATCH 11/28] backup warning message updated --- packages/contentstack-import/src/utils/backup-handler.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/contentstack-import/src/utils/backup-handler.ts b/packages/contentstack-import/src/utils/backup-handler.ts index 802a727896..bd76abf784 100755 --- a/packages/contentstack-import/src/utils/backup-handler.ts +++ b/packages/contentstack-import/src/utils/backup-handler.ts @@ -17,9 +17,12 @@ export default function setupBackupDir(importConfig: ImportConfig): Promise Date: Mon, 28 Aug 2023 12:10:46 +0530 Subject: [PATCH 12/28] added json rte fix --- .../contentstack-import/src/utils/content-type-helper.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/contentstack-import/src/utils/content-type-helper.ts b/packages/contentstack-import/src/utils/content-type-helper.ts index 3644cbc750..140576af8f 100644 --- a/packages/contentstack-import/src/utils/content-type-helper.ts +++ b/packages/contentstack-import/src/utils/content-type-helper.ts @@ -128,6 +128,15 @@ export const removeReferenceFields = async function ( }); } } + } else if ( + // handling entry references in json rte + schema[i].data_type === 'json' && + schema[i].field_metadata.allow_json_rte && + schema[i].field_metadata.embed_entry && + schema[i].reference_to.length > 1 + ) { + flag.supressed = true; + schema[i].reference_to = ['sys_assets']; } else if ( // handling entry references in json rte schema[i].data_type === 'json' && From 1883af5390d93a5b9d3fb5252f1830b9fb198a38 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 28 Aug 2023 13:08:46 +0530 Subject: [PATCH 13/28] refactor: enable environment rewrite --- package-lock.json | 87 +++++-------------- .../src/commands/cm/stacks/import.ts | 5 +- .../contentstack-import/src/config/index.ts | 1 + .../src/import/modules/base-class.ts | 12 +-- .../src/import/modules/labels.ts | 4 +- 5 files changed, 34 insertions(+), 75 deletions(-) diff --git a/package-lock.json b/package-lock.json index c881e9a818..bf9c499b51 100644 --- a/package-lock.json +++ b/package-lock.json @@ -807,26 +807,27 @@ } }, "node_modules/@contentstack/management": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.0.tgz", - "integrity": "sha512-wnmVS19n3cZeh6T2PbBXGCr4irktunVmuECTyPF5++Rsc6GDO+FoQGCZdvzHpRZE0GLiPFIID50z2TZuNArEOg==", + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.2.tgz", + "integrity": "sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==", "dependencies": { - "axios": "^0.27.2", + "axios": "^1.4.0", "form-data": "^3.0.1", "lodash": "^4.17.21", - "qs": "^6.10.3" + "qs": "^6.11.2" }, "engines": { "node": ">=8.0.0" } }, "node_modules/@contentstack/management/node_modules/axios": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", - "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", + "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", "dependencies": { - "follow-redirects": "^1.14.9", - "form-data": "^4.0.0" + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" } }, "node_modules/@contentstack/management/node_modules/axios/node_modules/form-data": { @@ -2049,14 +2050,6 @@ "node": ">=14.0.0" } }, - "node_modules/@oclif/core/node_modules/astral-regex": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", - "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", - "engines": { - "node": ">=8" - } - }, "node_modules/@oclif/core/node_modules/globby": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", @@ -2076,22 +2069,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@oclif/core/node_modules/slice-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", - "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", - "dependencies": { - "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0", - "is-fullwidth-code-point": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, "node_modules/@oclif/errors": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/@oclif/errors/-/errors-1.3.6.tgz", @@ -3895,7 +3872,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", - "dev": true, "engines": { "node": ">=8" } @@ -17082,7 +17058,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", @@ -25988,23 +25963,24 @@ } }, "@contentstack/management": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.0.tgz", - "integrity": "sha512-wnmVS19n3cZeh6T2PbBXGCr4irktunVmuECTyPF5++Rsc6GDO+FoQGCZdvzHpRZE0GLiPFIID50z2TZuNArEOg==", + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.2.tgz", + "integrity": "sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==", "requires": { - "axios": "^0.27.2", + "axios": "^1.4.0", "form-data": "^3.0.1", "lodash": "^4.17.21", - "qs": "^6.10.3" + "qs": "^6.11.2" }, "dependencies": { "axios": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", - "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", + "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", "requires": { - "follow-redirects": "^1.14.9", - "form-data": "^4.0.0" + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" }, "dependencies": { "form-data": { @@ -26994,11 +26970,6 @@ "wrap-ansi": "^7.0.0" }, "dependencies": { - "astral-regex": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", - "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==" - }, "globby": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", @@ -27011,16 +26982,6 @@ "merge2": "^1.4.1", "slash": "^3.0.0" } - }, - "slice-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", - "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", - "requires": { - "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0", - "is-fullwidth-code-point": "^3.0.0" - } } } }, @@ -28490,8 +28451,7 @@ "astral-regex": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", - "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", - "dev": true + "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==" }, "async": { "version": "3.2.4", @@ -38569,7 +38529,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", - "dev": true, "requires": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index 026ece969e..391ba9a136 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -110,18 +110,17 @@ export default class ImportCommand extends Command { // Note setting host to create cma client importConfig.host = this.cmaHost; contentDir = importConfig.contentDir; - backupDir = importConfig.backupDir; const managementAPIClient: ContentstackClient = await managementSDKClient(importConfig); const moduleImporter = new ModuleImporter(managementAPIClient, importConfig); await moduleImporter.start(); log(importConfig, `The content has been imported to the stack ${importConfig.apiKey} successfully!`, 'success'); - log(importConfig, `The log has been stored at '${path.join(importConfig.backupDir, 'logs', 'import')}'`, 'success'); + log(importConfig, `The log has been stored at '${path.join(contentDir, 'logs', 'import')}'`, 'success'); } catch (error) { log({ data: contentDir } as ImportConfig, `Failed to import stack content - ${formatError(error)}`, 'error'); log( { data: contentDir } as ImportConfig, `The log has been stored at ${ - { data: backupDir } ? path.join(backupDir || __dirname, 'logs', 'import') : path.join(__dirname, 'logs') + { data: contentDir } ? path.join(contentDir || __dirname, 'logs', 'import') : path.join(__dirname, 'logs') }`, 'info', ); diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index d1745a1288..2835c7747d 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -379,6 +379,7 @@ const config: DefaultConfig = { 'custom-roles', 'workflows', 'entries', + 'environments' ], rateLimit: 5, preserveStackVersion: false, diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 3510d8ae88..877844baba 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -13,7 +13,7 @@ import { FolderData } from '@contentstack/management/types/stack/asset/folder'; import { ExtensionData } from '@contentstack/management/types/stack/extension'; import { GlobalFieldData } from '@contentstack/management/types/stack/globalField'; import { ContentTypeData } from '@contentstack/management/types/stack/contentType'; -// import { EnvironmentData } from '@contentstack/management/types/stack/environment'; +import { EnvironmentData } from '@contentstack/management/types/stack/environment'; import { LabelData } from '@contentstack/management/types/stack/label'; import { WebhookData } from '@contentstack/management/types/stack/webhook'; import { WorkflowData } from '@contentstack/management/types/stack/workflow'; @@ -306,11 +306,11 @@ export default abstract class BaseClass { case 'update-gfs': return apiData.update().then(onSuccess).catch(onReject); case 'create-environments': - // return this.stack - // .environment() - // .create({ environment: omit(apiData, ['uid']) as EnvironmentData }) - // .then(onSuccess) - // .catch(onReject); + return this.stack + .environment() + .create({ environment: omit(apiData, ['uid']) as EnvironmentData }) + .then(onSuccess) + .catch(onReject); case 'create-labels': return this.stack .label() diff --git a/packages/contentstack-import/src/import/modules/labels.ts b/packages/contentstack-import/src/import/modules/labels.ts index 8db99db9ce..e109d3714b 100644 --- a/packages/contentstack-import/src/import/modules/labels.ts +++ b/packages/contentstack-import/src/import/modules/labels.ts @@ -152,7 +152,7 @@ export default class Importlabels extends BaseClass { apiContent, processName: 'update labels', apiParams: { - serializeData: this.serializeUpdatelabels.bind(this), + serializeData: this.serializeUpdateLabels.bind(this), reject: onReject.bind(this), resolve: onSuccess.bind(this), entity: 'update-labels', @@ -171,7 +171,7 @@ export default class Importlabels extends BaseClass { * @param {ApiOptions} apiOptions ApiOptions * @returns {ApiOptions} ApiOptions */ - serializeUpdatelabels(apiOptions: ApiOptions): ApiOptions { + serializeUpdateLabels(apiOptions: ApiOptions): ApiOptions { const { apiData: label } = apiOptions; const labelUid = label.uid; if (this.labelUidMapper.hasOwnProperty(labelUid)) { From 4eb52ee034c37c93fef0e9c96dbb7b0de4671a6a Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 28 Aug 2023 17:20:13 +0530 Subject: [PATCH 14/28] fix: management sdk version update --- package-lock.json | 12 +++---- packages/contentstack-import/package.json | 2 +- packages/contentstack-utilities/package.json | 2 +- packages/contentstack/package.json | 2 +- pnpm-lock.yaml | 33 ++++++++++---------- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index bf9c499b51..c6964eca18 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20293,7 +20293,7 @@ "@contentstack/cli-launch": "~1.0.10", "@contentstack/cli-migration": "~1.3.11", "@contentstack/cli-utilities": "~1.5.1", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.3.9", @@ -21305,7 +21305,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.11", "@contentstack/cli-utilities": "~1.5.1", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.2", "@oclif/config": "^1.18.3", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", @@ -22094,7 +22094,7 @@ "version": "1.5.1", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", @@ -23706,7 +23706,7 @@ "@contentstack/cli-launch": "~1.0.10", "@contentstack/cli-migration": "~1.3.11", "@contentstack/cli-utilities": "~1.5.1", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.3.9", @@ -24231,7 +24231,7 @@ "requires": { "@contentstack/cli-command": "~1.2.11", "@contentstack/cli-utilities": "~1.5.1", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.2", "@oclif/config": "^1.18.3", "@oclif/core": "^2.9.3", "@oclif/test": "^1.2.6", @@ -25109,7 +25109,7 @@ "@contentstack/cli-utilities": { "version": "file:packages/contentstack-utilities", "requires": { - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "@oclif/test": "^2.2.10", "@types/chai": "^4.2.18", diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index ceae4f1c06..5fa29bd959 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -7,7 +7,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.11", "@contentstack/cli-utilities": "~1.5.1", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.2", "@oclif/config": "^1.18.3", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", diff --git a/packages/contentstack-utilities/package.json b/packages/contentstack-utilities/package.json index 8b71070971..7bb46ca2d6 100644 --- a/packages/contentstack-utilities/package.json +++ b/packages/contentstack-utilities/package.json @@ -32,7 +32,7 @@ "author": "contentstack", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", diff --git a/packages/contentstack/package.json b/packages/contentstack/package.json index 14e61aab08..adf54944b0 100755 --- a/packages/contentstack/package.json +++ b/packages/contentstack/package.json @@ -36,7 +36,7 @@ "@contentstack/cli-launch": "~1.0.10", "@contentstack/cli-migration": "~1.3.11", "@contentstack/cli-utilities": "~1.5.1", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.2", "@contentstack/cli-cm-branches": "~1.0.11", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.3.9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5cdc9f26f5..c48a9f8006 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,7 +25,7 @@ importers: '@contentstack/cli-launch': ~1.0.10 '@contentstack/cli-migration': ~1.3.11 '@contentstack/cli-utilities': ~1.5.1 - '@contentstack/management': ~1.10.0 + '@contentstack/management': ~1.10.2 '@oclif/core': ^2.9.3 '@oclif/plugin-help': ^5 '@oclif/plugin-not-found': ^2.3.9 @@ -78,7 +78,7 @@ importers: '@contentstack/cli-launch': link:../contentstack-launch '@contentstack/cli-migration': link:../contentstack-migration '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.10.0_debug@4.3.4 + '@contentstack/management': 1.10.2_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-help': 5.2.14_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-not-found': 2.3.32_4qcp7qp4jxxdgb4qbxgwox4hwq @@ -659,7 +659,7 @@ importers: specifiers: '@contentstack/cli-command': ~1.2.11 '@contentstack/cli-utilities': ~1.5.1 - '@contentstack/management': ~1.10.0 + '@contentstack/management': ~1.10.2 '@oclif/config': ^1.18.3 '@oclif/core': ^2.9.3 '@oclif/test': ^1.2.6 @@ -699,7 +699,7 @@ importers: dependencies: '@contentstack/cli-command': link:../contentstack-command '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.10.0_debug@4.3.4 + '@contentstack/management': 1.10.2_debug@4.3.4 '@oclif/config': 1.18.14 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq big-json: 3.2.0 @@ -970,7 +970,7 @@ importers: packages/contentstack-utilities: specifiers: - '@contentstack/management': ~1.10.0 + '@contentstack/management': ~1.10.2 '@oclif/core': ^2.9.3 '@oclif/test': ^2.2.10 '@types/chai': ^4.2.18 @@ -1014,7 +1014,7 @@ importers: winston: ^3.7.2 xdg-basedir: ^4.0.0 dependencies: - '@contentstack/management': 1.10.0_debug@4.3.4 + '@contentstack/management': 1.10.2_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq axios: 1.3.4_debug@4.3.4 chalk: 4.1.2 @@ -1501,11 +1501,11 @@ packages: uuid: 8.3.2 dev: false - /@contentstack/management/1.10.0_debug@4.3.4: - resolution: {integrity: sha512-wnmVS19n3cZeh6T2PbBXGCr4irktunVmuECTyPF5++Rsc6GDO+FoQGCZdvzHpRZE0GLiPFIID50z2TZuNArEOg==} + /@contentstack/management/1.10.2_debug@4.3.4: + resolution: {integrity: sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==} engines: {node: '>=8.0.0'} dependencies: - axios: 0.27.2_debug@4.3.4 + axios: 1.5.0_debug@4.3.4 form-data: 3.0.1 lodash: 4.17.21 qs: 6.11.2 @@ -4193,27 +4193,28 @@ packages: xml2js: 0.5.0 dev: true - /axios/0.27.2_debug@4.3.4: - resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} + /axios/1.3.4: + resolution: {integrity: sha512-toYm+Bsyl6VC5wSkfkbbNB6ROv7KY93PEBBL6xyDczaIHasAiv4wPqQ/c4RjoQzipxRD2W5g21cOqQulZ7rHwQ==} dependencies: - follow-redirects: 1.15.2_debug@4.3.4 + follow-redirects: 1.15.2 form-data: 4.0.0 + proxy-from-env: 1.1.0 transitivePeerDependencies: - debug dev: false - /axios/1.3.4: + /axios/1.3.4_debug@4.3.4: resolution: {integrity: sha512-toYm+Bsyl6VC5wSkfkbbNB6ROv7KY93PEBBL6xyDczaIHasAiv4wPqQ/c4RjoQzipxRD2W5g21cOqQulZ7rHwQ==} dependencies: - follow-redirects: 1.15.2 + follow-redirects: 1.15.2_debug@4.3.4 form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug dev: false - /axios/1.3.4_debug@4.3.4: - resolution: {integrity: sha512-toYm+Bsyl6VC5wSkfkbbNB6ROv7KY93PEBBL6xyDczaIHasAiv4wPqQ/c4RjoQzipxRD2W5g21cOqQulZ7rHwQ==} + /axios/1.5.0_debug@4.3.4: + resolution: {integrity: sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==} dependencies: follow-redirects: 1.15.2_debug@4.3.4 form-data: 4.0.0 From dcf7d01272816f729a8e4601dbbdff372828125a Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Mon, 28 Aug 2023 19:27:10 +0530 Subject: [PATCH 15/28] export import config update --- package-lock.json | 22 +++++++++++++++++++ packages/contentstack-auth/README.md | 2 +- packages/contentstack-bootstrap/README.md | 2 +- packages/contentstack-branches/README.md | 2 +- packages/contentstack-bulk-publish/README.md | 2 +- packages/contentstack-clone/README.md | 2 +- packages/contentstack-config/README.md | 2 +- packages/contentstack-export/README.md | 2 +- packages/contentstack-export/package.json | 4 ++-- .../src/commands/cm/stacks/export.ts | 3 ++- .../contentstack-export/src/config/index.ts | 16 +------------- .../src/export/module-exporter.ts | 3 ++- .../src/types/default-config.ts | 3 +-- .../src/utils/common-helper.ts | 11 ++++++++-- .../src/utils/file-helper.ts | 2 +- .../contentstack-export/src/utils/logger.ts | 6 ++--- packages/contentstack-export/tsconfig.json | 2 +- packages/contentstack-import/README.md | 2 +- packages/contentstack-import/package.json | 1 + .../contentstack-import/src/config/index.ts | 14 ------------ .../src/import/module-importer.ts | 20 +++++++---------- .../src/types/default-config.ts | 2 -- .../src/types/import-config.ts | 3 ++- .../src/utils/file-helper.ts | 8 +++---- .../src/utils/import-config-handler.ts | 9 +++++++- packages/contentstack-import/tsconfig.json | 2 +- packages/contentstack-launch/README.md | 2 +- packages/contentstack-migrate-rte/README.md | 2 +- packages/contentstack-migration/README.md | 2 +- packages/contentstack/README.md | 2 +- 30 files changed, 80 insertions(+), 75 deletions(-) diff --git a/package-lock.json b/package-lock.json index c881e9a818..822d8ccdd3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2693,6 +2693,15 @@ "@babel/types": "^7.20.7" } }, + "node_modules/@types/big-json": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@types/big-json/-/big-json-3.2.0.tgz", + "integrity": "sha512-2nV9P72HH6FAoxoJLIKxNGLSX6pM9nm1YiOfQQwr7Sy3oz2rF4RRsxmaB7T/tcESecAqM6f0CYjx6GqnrN5X2Q==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/bluebird": { "version": "3.5.38", "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.38.tgz", @@ -21052,6 +21061,7 @@ "@contentstack/cli-dev-dependencies": "~1.2.3", "@oclif/plugin-help": "^5.1.19", "@oclif/test": "^1.2.6", + "@types/big-json": "^3.2.0", "@types/mkdirp": "^1.0.2", "@types/progress-stream": "^2.0.2", "assert": "^2.0.0", @@ -21348,6 +21358,7 @@ }, "devDependencies": { "@oclif/test": "^1.2.6", + "@types/big-json": "^3.2.0", "@types/bluebird": "^3.5.38", "@types/chai": "^4.2.18", "@types/fs-extra": "^11.0.1", @@ -24020,6 +24031,7 @@ "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5.1.19", "@oclif/test": "^1.2.6", + "@types/big-json": "^3.2.0", "@types/mkdirp": "^1.0.2", "@types/progress-stream": "^2.0.2", "assert": "^2.0.0", @@ -24260,6 +24272,7 @@ "@oclif/config": "^1.18.3", "@oclif/core": "^2.9.3", "@oclif/test": "^1.2.6", + "@types/big-json": "^3.2.0", "@types/bluebird": "^3.5.38", "@types/chai": "^4.2.18", "@types/fs-extra": "^11.0.1", @@ -27538,6 +27551,15 @@ "@babel/types": "^7.20.7" } }, + "@types/big-json": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@types/big-json/-/big-json-3.2.0.tgz", + "integrity": "sha512-2nV9P72HH6FAoxoJLIKxNGLSX6pM9nm1YiOfQQwr7Sy3oz2rF4RRsxmaB7T/tcESecAqM6f0CYjx6GqnrN5X2Q==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/bluebird": { "version": "3.5.38", "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.38.tgz", diff --git a/packages/contentstack-auth/README.md b/packages/contentstack-auth/README.md index 3ca095069f..2f2c47b143 100644 --- a/packages/contentstack-auth/README.md +++ b/packages/contentstack-auth/README.md @@ -18,7 +18,7 @@ $ npm install -g @contentstack/cli-auth $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-auth/1.3.12 darwin-arm64 node-v20.3.1 +@contentstack/cli-auth/1.3.12 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-bootstrap/README.md b/packages/contentstack-bootstrap/README.md index 993ef9d09c..c7a8e634fb 100644 --- a/packages/contentstack-bootstrap/README.md +++ b/packages/contentstack-bootstrap/README.md @@ -15,7 +15,7 @@ $ npm install -g @contentstack/cli-cm-bootstrap $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-bootstrap/1.4.14 darwin-arm64 node-v20.3.1 +@contentstack/cli-cm-bootstrap/1.4.14 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-branches/README.md b/packages/contentstack-branches/README.md index d6c9aeb884..d961644214 100755 --- a/packages/contentstack-branches/README.md +++ b/packages/contentstack-branches/README.md @@ -37,7 +37,7 @@ $ npm install -g @contentstack/cli-cm-branches $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-branches/1.0.11 darwin-arm64 node-v20.3.1 +@contentstack/cli-cm-branches/1.0.11 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-bulk-publish/README.md b/packages/contentstack-bulk-publish/README.md index c7295eccb1..bb323bc369 100644 --- a/packages/contentstack-bulk-publish/README.md +++ b/packages/contentstack-bulk-publish/README.md @@ -18,7 +18,7 @@ $ npm install -g @contentstack/cli-cm-bulk-publish $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-bulk-publish/1.3.10 darwin-arm64 node-v20.3.1 +@contentstack/cli-cm-bulk-publish/1.3.10 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-clone/README.md b/packages/contentstack-clone/README.md index f270ae94b7..a18e49330c 100644 --- a/packages/contentstack-clone/README.md +++ b/packages/contentstack-clone/README.md @@ -16,7 +16,7 @@ $ npm install -g @contentstack/cli-cm-clone $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-clone/1.4.15 darwin-arm64 node-v20.3.1 +@contentstack/cli-cm-clone/1.4.15 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-config/README.md b/packages/contentstack-config/README.md index 831a9ddd87..a8715d82b1 100644 --- a/packages/contentstack-config/README.md +++ b/packages/contentstack-config/README.md @@ -18,7 +18,7 @@ $ npm install -g @contentstack/cli-config $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-config/1.4.10 darwin-arm64 node-v20.3.1 +@contentstack/cli-config/1.4.10 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-export/README.md b/packages/contentstack-export/README.md index 7d803b6478..b78b5f62df 100755 --- a/packages/contentstack-export/README.md +++ b/packages/contentstack-export/README.md @@ -48,7 +48,7 @@ $ npm install -g @contentstack/cli-cm-export $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-export/1.8.0 darwin-arm64 node-v20.3.1 +@contentstack/cli-cm-export/1.8.0 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-export/package.json b/packages/contentstack-export/package.json index 45aac8ab41..d06e4dfc09 100644 --- a/packages/contentstack-export/package.json +++ b/packages/contentstack-export/package.json @@ -31,9 +31,9 @@ "@contentstack/cli-dev-dependencies": "~1.2.3", "@oclif/plugin-help": "^5.1.19", "@oclif/test": "^1.2.6", + "@types/big-json": "^3.2.0", "@types/mkdirp": "^1.0.2", "@types/progress-stream": "^2.0.2", - "@types/mkdirp": "^1.0.2", "assert": "^2.0.0", "chai": "^4.2.0", "dotenv": "^16.0.1", @@ -100,4 +100,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} \ No newline at end of file +} diff --git a/packages/contentstack-export/src/commands/cm/stacks/export.ts b/packages/contentstack-export/src/commands/cm/stacks/export.ts index 60cef825da..24eb5cde66 100644 --- a/packages/contentstack-export/src/commands/cm/stacks/export.ts +++ b/packages/contentstack-export/src/commands/cm/stacks/export.ts @@ -10,7 +10,7 @@ import { FlagInput, } from '@contentstack/cli-utilities'; import { ModuleExporter } from '../../../export'; -import { setupExportConfig, log, formatError } from '../../../utils'; +import { setupExportConfig, log, formatError, writeExportMetaFile } from '../../../utils'; import { ExportConfig } from '../../../types'; export default class ExportCommand extends Command { @@ -110,6 +110,7 @@ export default class ExportCommand extends Command { const managementAPIClient: ContentstackClient = await managementSDKClient(exportConfig); const moduleExporter = new ModuleExporter(managementAPIClient, exportConfig); await moduleExporter.start(); + writeExportMetaFile(exportConfig); log(exportConfig, `The content of the stack ${exportConfig.apiKey} has been exported successfully!`, 'success'); log(exportConfig, `The log has been stored at '${path.join(exportDir, 'logs', 'export')}'`, 'success'); } catch (error) { diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 9a98fdcc8a..d2b3e3a199 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -1,6 +1,7 @@ import { DefaultConfig } from '../types'; const config: DefaultConfig = { + version: 1, versioning: false, host: 'https://api.contentstack.io/v3', developerHubUrls: { @@ -369,20 +370,6 @@ const config: DefaultConfig = { 'xh', 'zu', ], - updatedModules: [ - 'assets', - 'locales', - 'extensions', - 'marketplace-apps', - 'labels', - 'webhooks', - 'custom-roles', - 'global-fields', - 'content-types', - 'entries', - 'workflows', - 'stack', - ], apis: { userSession: '/user-session/', globalfields: '/global_fields/', @@ -402,7 +389,6 @@ const config: DefaultConfig = { writeConcurrency: 5, developerHubBaseUrl: '', marketplaceAppEncryptionKey: 'nF2ejRQcTv', - useNewModuleStructure: true, }; export default config; diff --git a/packages/contentstack-export/src/export/module-exporter.ts b/packages/contentstack-export/src/export/module-exporter.ts index a404af4570..170dfe029e 100644 --- a/packages/contentstack-export/src/export/module-exporter.ts +++ b/packages/contentstack-export/src/export/module-exporter.ts @@ -47,6 +47,7 @@ class ModuleExporter { } async export() { + log(this.exportConfig, `Started to export content, version is ${this.exportConfig.version}`, 'info'); // checks for single module or all modules if (this.exportConfig.singleModuleExport) { return this.exportSingleModule(this.exportConfig.moduleName); @@ -59,7 +60,7 @@ class ModuleExporter { // export the modules by name // calls the module runner which inturn calls the module itself let exportedModuleResponse; - if (this.exportConfig.useNewModuleStructure && this.exportConfig.updatedModules.indexOf(moduleName) !== -1) { + if (this.exportConfig.version === 2) { exportedModuleResponse = await startModuleExport({ stackAPIClient: this.stackAPIClient, exportConfig: this.exportConfig, diff --git a/packages/contentstack-export/src/types/default-config.ts b/packages/contentstack-export/src/types/default-config.ts index 918786bdac..5ba540a3ef 100644 --- a/packages/contentstack-export/src/types/default-config.ts +++ b/packages/contentstack-export/src/types/default-config.ts @@ -1,6 +1,7 @@ import { Modules } from '.'; export default interface DefaultConfig { + version: number; versioning: boolean; host: string; cdn?: string; @@ -145,7 +146,6 @@ export default interface DefaultConfig { }; }; languagesCode: string[]; - updatedModules: string[]; apis: { userSession: string; globalfields: string; @@ -165,5 +165,4 @@ export default interface DefaultConfig { writeConcurrency: number; developerHubBaseUrl: string; marketplaceAppEncryptionKey: string; - useNewModuleStructure: boolean; } diff --git a/packages/contentstack-export/src/utils/common-helper.ts b/packages/contentstack-export/src/utils/common-helper.ts index fc5327d070..523774d6e1 100644 --- a/packages/contentstack-export/src/utils/common-helper.ts +++ b/packages/contentstack-export/src/utils/common-helper.ts @@ -5,9 +5,10 @@ */ import promiseLimit from 'promise-limit'; +import * as path from 'path'; import { isAuthenticated } from '@contentstack/cli-utilities'; - -import { ExternalConfig } from '../types'; +import { ExternalConfig, ExportConfig } from '../types'; +import { fsUtil } from './file-helper'; export const validateConfig = function (config: ExternalConfig) { if (!config.host || !config.cdn) { @@ -77,3 +78,9 @@ export const executeTask = function ( }), ); }; + +// Note: we can add more useful details in meta file +export const writeExportMetaFile = (exportConfig: ExportConfig) => { + const exportMeta = { version: exportConfig.version, logsPath: path.join(exportConfig.exportDir, 'logs', 'export') }; + fsUtil.writeFile(path.join(exportConfig.exportDir, 'export-info.json'), exportMeta); +}; diff --git a/packages/contentstack-export/src/utils/file-helper.ts b/packages/contentstack-export/src/utils/file-helper.ts index cc9863c843..ce7c7f326e 100644 --- a/packages/contentstack-export/src/utils/file-helper.ts +++ b/packages/contentstack-export/src/utils/file-helper.ts @@ -50,7 +50,7 @@ export const readLargeFile = function (filePath: string, options: { type?: strin console.log('error', error); reject(error); }); - readStream.pipe(parseStream); + readStream.pipe(parseStream as any); }); } }; diff --git a/packages/contentstack-export/src/utils/logger.ts b/packages/contentstack-export/src/utils/logger.ts index e808f2dbdf..c3edf2ce27 100644 --- a/packages/contentstack-export/src/utils/logger.ts +++ b/packages/contentstack-export/src/utils/logger.ts @@ -134,13 +134,13 @@ function init(_logPath: string) { } export const log = async (config: ExportConfig, message: any, type: string) => { - config.data = config.data || path.join(__dirname, 'logs'); + const logsPath = config.data || __dirname; // ignoring the type argument, as we are not using it to create a logfile anymore if (type !== 'error') { // removed type argument from init method - init(config.data).log(message); + init(logsPath).log(message); } else { - init(config.data).error(message); + init(logsPath).error(message); } }; diff --git a/packages/contentstack-export/tsconfig.json b/packages/contentstack-export/tsconfig.json index da8fc4b724..5136623392 100644 --- a/packages/contentstack-export/tsconfig.json +++ b/packages/contentstack-export/tsconfig.json @@ -12,7 +12,7 @@ "skipLibCheck": true, "sourceMap": false, "esModuleInterop": true, - "noImplicitAny": false, + "noImplicitAny": true, "lib": [ "ES2019", "es2020.promise" diff --git a/packages/contentstack-import/README.md b/packages/contentstack-import/README.md index 20b53af987..c67771a6a0 100644 --- a/packages/contentstack-import/README.md +++ b/packages/contentstack-import/README.md @@ -47,7 +47,7 @@ $ npm install -g @contentstack/cli-cm-import $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-import/1.8.2 darwin-arm64 node-v20.3.1 +@contentstack/cli-cm-import/1.8.2 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index ceae4f1c06..0429e1bce4 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -25,6 +25,7 @@ }, "devDependencies": { "@oclif/test": "^1.2.6", + "@types/big-json": "^3.2.0", "@types/bluebird": "^3.5.38", "@types/chai": "^4.2.18", "@types/fs-extra": "^11.0.1", diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index d1745a1288..faf4dd43ca 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -367,19 +367,6 @@ const config: DefaultConfig = { stacks: '/stacks/', labels: '/labels/', }, - updatedModules: [ - 'assets', - 'extensions', - 'locales', - 'marketplace-apps', - 'labels', - 'global-fields', - 'content-types', - 'webhooks', - 'custom-roles', - 'workflows', - 'entries', - ], rateLimit: 5, preserveStackVersion: false, entriesPublish: true, @@ -390,7 +377,6 @@ const config: DefaultConfig = { developerHubBaseUrl: '', marketplaceAppEncryptionKey: 'nF2ejRQcTv', getEncryptionKeyMaxRetry: 3, - useNewModuleStructure: true, // useBackedupDir: '', // backupConcurrency: 10, }; diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index df808bfbe0..921fe4a0e6 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -60,6 +60,8 @@ class ModuleImporter { } async import() { + log(this.importConfig, `Starting to import content ${this.importConfig.contentVersion}`, 'info'); + // checks for single module or all modules if (this.importConfig.singleModuleImport) { return this.importByModuleByName(this.importConfig.moduleName); @@ -69,27 +71,21 @@ class ModuleImporter { async importByModuleByName(moduleName: Modules) { log(this.importConfig, `Starting import of ${moduleName} module`, 'info'); - - const basePath = `${this.importConfig.backupDir}/${moduleName}`; // import the modules by name // calls the module runner which inturn calls the module itself // Todo: Implement a mechanism to determine whether module is new or old - if ( - this.importConfig.useNewModuleStructure && - this.importConfig.updatedModules.indexOf(moduleName) !== -1 - //&& new FsUtility({ basePath }).isNewFsStructure - ) { + if (this.importConfig.contentVersion === 2) { return startModuleImport({ stackAPIClient: this.stackAPIClient, importConfig: this.importConfig, moduleName, }); } - return startJSModuleImport({ - stackAPIClient: this.stackAPIClient, - importConfig: this.importConfig, - moduleName, - }); + return startJSModuleImport({ + stackAPIClient: this.stackAPIClient, + importConfig: this.importConfig, + moduleName, + }); } async importAllModules(): Promise { diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index b833ed90fa..c41b949fdb 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -129,7 +129,6 @@ export default interface DefaultConfig { stacks: string; labels: string; }; - updatedModules: string[]; rateLimit: number; preserveStackVersion: boolean; entriesPublish: boolean; @@ -140,6 +139,5 @@ export default interface DefaultConfig { developerHubBaseUrl: string; marketplaceAppEncryptionKey: string; getEncryptionKeyMaxRetry: number; - useNewModuleStructure: boolean; createBackupDir?: string; } diff --git a/packages/contentstack-import/src/types/import-config.ts b/packages/contentstack-import/src/types/import-config.ts index d857551fac..1609d00df9 100644 --- a/packages/contentstack-import/src/types/import-config.ts +++ b/packages/contentstack-import/src/types/import-config.ts @@ -42,7 +42,8 @@ export default interface ImportConfig extends DefaultConfig, ExternalConfig { backupConcurrency?: number; authtoken?: string; destinationStackName?: string; - org_uid?:string; + org_uid?: string; + contentVersion: number; } type branch = { diff --git a/packages/contentstack-import/src/utils/file-helper.ts b/packages/contentstack-import/src/utils/file-helper.ts index dd97ea0dc4..a8446e93cb 100644 --- a/packages/contentstack-import/src/utils/file-helper.ts +++ b/packages/contentstack-import/src/utils/file-helper.ts @@ -56,7 +56,7 @@ export const readLargeFile = function (filePath: string, opts?: any): Promise => { } config = merge.recursive(config, externalConfig); } + config.contentDir = importCmdFlags['data'] || importCmdFlags['data-dir'] || config.data || (await askContentDir()); config.contentDir = path.resolve(config.contentDir); //Note to support the old key config.data = config.contentDir; + if (fileExistsSync(path.join(config.contentDir, 'export-info.json'))) { + config.contentVersion = ((await readFile(path.join(config.contentDir, 'export-info.json'))) || {}).version || 2; + } else { + config.contentVersion = 1; + } + const managementTokenAlias = importCmdFlags['management-token-alias'] || importCmdFlags['alias']; if (managementTokenAlias) { diff --git a/packages/contentstack-import/tsconfig.json b/packages/contentstack-import/tsconfig.json index 468adb221f..353cb22a4f 100644 --- a/packages/contentstack-import/tsconfig.json +++ b/packages/contentstack-import/tsconfig.json @@ -12,7 +12,7 @@ "skipLibCheck": true, "sourceMap": false, "esModuleInterop": true, - "noImplicitAny": false, + "noImplicitAny": true, "lib": [ "ES2019", "es2020.promise" diff --git a/packages/contentstack-launch/README.md b/packages/contentstack-launch/README.md index 02c7a6cc48..018e169057 100755 --- a/packages/contentstack-launch/README.md +++ b/packages/contentstack-launch/README.md @@ -19,7 +19,7 @@ $ npm install -g @contentstack/cli-launch $ csdx COMMAND running command... $ csdx (--version|-v) -@contentstack/cli-launch/1.0.10 darwin-arm64 node-v20.3.1 +@contentstack/cli-launch/1.0.10 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-migrate-rte/README.md b/packages/contentstack-migrate-rte/README.md index 77f51af25e..dfd4e10f39 100644 --- a/packages/contentstack-migrate-rte/README.md +++ b/packages/contentstack-migrate-rte/README.md @@ -16,7 +16,7 @@ $ npm install -g @contentstack/cli-cm-migrate-rte $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-migrate-rte/1.4.10 darwin-arm64 node-v20.3.1 +@contentstack/cli-cm-migrate-rte/1.4.10 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-migration/README.md b/packages/contentstack-migration/README.md index 23933f2703..4dfd6b8b81 100644 --- a/packages/contentstack-migration/README.md +++ b/packages/contentstack-migration/README.md @@ -21,7 +21,7 @@ $ npm install -g @contentstack/cli-migration $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-migration/1.3.11 darwin-arm64 node-v20.3.1 +@contentstack/cli-migration/1.3.11 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack/README.md b/packages/contentstack/README.md index 71f2806b2b..aa7be6a19d 100644 --- a/packages/contentstack/README.md +++ b/packages/contentstack/README.md @@ -18,7 +18,7 @@ $ npm install -g @contentstack/cli $ csdx COMMAND running command... $ csdx (--version|-v) -@contentstack/cli/1.8.2 darwin-arm64 node-v20.3.1 +@contentstack/cli/1.8.2 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND From 29bbf0ba9ce4f25efa93b9357e4db95f494bc21c Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Tue, 29 Aug 2023 11:06:01 +0530 Subject: [PATCH 16/28] config updates --- .../example_config/auth_config.json | 6 +----- .../example_config/management_config.json | 6 +----- pnpm-lock.yaml | 10 ++++++++++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/contentstack-export/example_config/auth_config.json b/packages/contentstack-export/example_config/auth_config.json index 4e4743f721..6def43821c 100644 --- a/packages/contentstack-export/example_config/auth_config.json +++ b/packages/contentstack-export/example_config/auth_config.json @@ -1,4 +1,5 @@ { + "version": 2, "master_locale": { "name": "English - United States", "code": "en-us" @@ -9,10 +10,5 @@ "moduleName": "content-types", "fetchConcurrency": 5, "writeConcurrency": 5, - "updatedModules": [ - "assets", - "locales" - ], - "useNewModuleStructure": true, "securedAssets": false } \ No newline at end of file diff --git a/packages/contentstack-export/example_config/management_config.json b/packages/contentstack-export/example_config/management_config.json index d2dd367007..3730fbd882 100644 --- a/packages/contentstack-export/example_config/management_config.json +++ b/packages/contentstack-export/example_config/management_config.json @@ -1,4 +1,5 @@ { + "version": 2, "master_locale": { "name": "English - United States", "code": "en-us" @@ -8,11 +9,6 @@ "moduleName": "content-types", "fetchConcurrency": 5, "writeConcurrency": 5, - "updatedModules": [ - "assets", - "locales" - ], - "useNewModuleStructure": true, "securedAssets": false, "createBackupDir": "./temp" } \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5cdc9f26f5..46869f03ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -547,6 +547,7 @@ importers: '@oclif/core': ^2.9.3 '@oclif/plugin-help': ^5.1.19 '@oclif/test': ^1.2.6 + '@types/big-json': ^3.2.0 '@types/mkdirp': ^1.0.2 '@types/progress-stream': ^2.0.2 assert: ^2.0.0 @@ -602,6 +603,7 @@ importers: '@contentstack/cli-dev-dependencies': link:../contentstack-dev-dependencies '@oclif/plugin-help': 5.2.14_typescript@4.9.5 '@oclif/test': 1.2.9 + '@types/big-json': 3.2.0 '@types/mkdirp': 1.0.2 '@types/progress-stream': 2.0.2 assert: 2.0.0 @@ -663,6 +665,7 @@ importers: '@oclif/config': ^1.18.3 '@oclif/core': ^2.9.3 '@oclif/test': ^1.2.6 + '@types/big-json': ^3.2.0 '@types/bluebird': ^3.5.38 '@types/chai': ^4.2.18 '@types/fs-extra': ^11.0.1 @@ -716,6 +719,7 @@ importers: winston: 3.10.0 devDependencies: '@oclif/test': 1.2.9 + '@types/big-json': 3.2.0 '@types/bluebird': 3.5.38 '@types/chai': 4.3.5 '@types/fs-extra': 11.0.1 @@ -3152,6 +3156,12 @@ packages: '@babel/types': 7.22.5 dev: true + /@types/big-json/3.2.0: + resolution: {integrity: sha512-2nV9P72HH6FAoxoJLIKxNGLSX6pM9nm1YiOfQQwr7Sy3oz2rF4RRsxmaB7T/tcESecAqM6f0CYjx6GqnrN5X2Q==} + dependencies: + '@types/node': 16.18.38 + dev: true + /@types/bluebird/3.5.38: resolution: {integrity: sha512-yR/Kxc0dd4FfwtEoLZMoqJbM/VE/W7hXn/MIjb+axcwag0iFmSPK7OBUZq1YWLynJUoWQkfUrI7T0HDqGApNSg==} dev: true From d0420080a680c06dc7d222b6c46e83efb8a69801 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 29 Aug 2023 11:35:01 +0530 Subject: [PATCH 17/28] fix: import log path & handle errors --- .../src/commands/cm/stacks/import.ts | 9 +++++---- .../src/import/module-importer.ts | 2 +- .../src/import/modules/custom-roles.ts | 12 +++++++++--- .../src/import/modules/entries.ts | 2 +- .../src/import/modules/workflows.ts | 3 ++- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/contentstack-import/src/commands/cm/stacks/import.ts b/packages/contentstack-import/src/commands/cm/stacks/import.ts index 391ba9a136..2712ca7b5c 100644 --- a/packages/contentstack-import/src/commands/cm/stacks/import.ts +++ b/packages/contentstack-import/src/commands/cm/stacks/import.ts @@ -110,17 +110,18 @@ export default class ImportCommand extends Command { // Note setting host to create cma client importConfig.host = this.cmaHost; contentDir = importConfig.contentDir; + backupDir = importConfig.backupDir; const managementAPIClient: ContentstackClient = await managementSDKClient(importConfig); const moduleImporter = new ModuleImporter(managementAPIClient, importConfig); await moduleImporter.start(); log(importConfig, `The content has been imported to the stack ${importConfig.apiKey} successfully!`, 'success'); - log(importConfig, `The log has been stored at '${path.join(contentDir, 'logs', 'import')}'`, 'success'); + log(importConfig, `The log has been stored at '${path.join(importConfig.backupDir, 'logs', 'import')}'`, 'success'); } catch (error) { - log({ data: contentDir } as ImportConfig, `Failed to import stack content - ${formatError(error)}`, 'error'); + log({ data: backupDir } as ImportConfig, `Failed to import stack content - ${formatError(error)}`, 'error'); log( - { data: contentDir } as ImportConfig, + { data: backupDir } as ImportConfig, `The log has been stored at ${ - { data: contentDir } ? path.join(contentDir || __dirname, 'logs', 'import') : path.join(__dirname, 'logs') + { data: backupDir } ? path.join(backupDir || __dirname, 'logs', 'import') : path.join(__dirname, 'logs') }`, 'info', ); diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index df808bfbe0..9dec284f00 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -48,10 +48,10 @@ class ModuleImporter { } const backupDir = await backupHandler(this.importConfig); if (backupDir) { - log(this.importConfig, `Backup directory path - '${backupDir}'`, 'info'); this.importConfig.backupDir = backupDir; // To support the old config this.importConfig.data = backupDir; + log(this.importConfig, `Backup directory path - '${backupDir}'`, 'info'); } await sanitizeStack(this.stackAPIClient); diff --git a/packages/contentstack-import/src/import/modules/custom-roles.ts b/packages/contentstack-import/src/import/modules/custom-roles.ts index 643812a8dd..05e344b699 100644 --- a/packages/contentstack-import/src/import/modules/custom-roles.ts +++ b/packages/contentstack-import/src/import/modules/custom-roles.ts @@ -198,11 +198,17 @@ export default class ImportCustomRoles extends BaseClass { getTransformUidsFactory = (rule: Record) => { if (rule.module === 'environment') { - rule.environments = map(rule.environments, (env: any) => this.environmentsUidMap[env]); + if(!isEmpty(this.environmentsUidMap)){ + rule.environments = map(rule.environments, (env: any) => this.environmentsUidMap[env]); + } } else if (rule.module === 'locale') { - rule.locales = map(rule.locales, (locale: any) => this.localesUidMap[locale]); + if(!isEmpty(this.localesUidMap)){ + rule.locales = map(rule.locales, (locale: any) => this.localesUidMap[locale]); + } } else if (rule.module === 'entry') { - rule.entries = map(rule.entries, (entry: any) => this.entriesUidMap[entry]); + if(!isEmpty(this.entriesUidMap)){ + rule.entries = map(rule.entries, (entry: any) => this.entriesUidMap[entry]); + } } return rule; }; diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 1188d18f90..2e80e0f8de 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -593,7 +593,7 @@ export default class EntriesImport extends BaseClass { }); log(this.importConfig, `Updated the field rules of ${cTUid}`, 'info'); } else { - log(this.importConfig, `No field rules found in content type ${cTUid} to update`, 'error'); + log(this.importConfig, `No field rules found in content type ${cTUid} to update`, 'info'); } } } diff --git a/packages/contentstack-import/src/import/modules/workflows.ts b/packages/contentstack-import/src/import/modules/workflows.ts index c1ab96975a..d403f71017 100644 --- a/packages/contentstack-import/src/import/modules/workflows.ts +++ b/packages/contentstack-import/src/import/modules/workflows.ts @@ -132,7 +132,8 @@ export default class ImportWorkflows extends BaseClass { const onReject = ({ error, apiData }: any) => { const err = error?.message ? JSON.parse(error.message) : error; const { name } = apiData; - if (err?.errors?.name) { + const workflowExists = err?.errors?.name || err?.errors?.['workflow.name']; + if (workflowExists) { log(this.importConfig, `Workflow '${name}' already exists`, 'info'); } else { this.failedWebhooks.push(apiData); From a6b35759bf67a7797f6302da72b0247cde6fbdec Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Tue, 29 Aug 2023 11:51:09 +0530 Subject: [PATCH 18/28] message updated --- packages/contentstack-import/src/import/module-importer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index 921fe4a0e6..d644b914b5 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -60,7 +60,7 @@ class ModuleImporter { } async import() { - log(this.importConfig, `Starting to import content ${this.importConfig.contentVersion}`, 'info'); + log(this.importConfig, `Starting to import content version ${this.importConfig.contentVersion}`, 'info'); // checks for single module or all modules if (this.importConfig.singleModuleImport) { From d0618bc2791e3bb25cb4f152e8ea750a162f356b Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 29 Aug 2023 16:23:14 +0530 Subject: [PATCH 19/28] refactor: enable environment new structure in export --- packages/contentstack-export/src/config/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 9a98fdcc8a..7bd3f4f85d 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -382,6 +382,7 @@ const config: DefaultConfig = { 'entries', 'workflows', 'stack', + 'environments' ], apis: { userSession: '/user-session/', From 8a7a45166c6c336819bfe5d853dfa055d9b7e473 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 29 Aug 2023 17:29:30 +0530 Subject: [PATCH 20/28] refactor: remove unnecessary log from module-exporter --- packages/contentstack-import/src/import/module-importer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/contentstack-import/src/import/module-importer.ts b/packages/contentstack-import/src/import/module-importer.ts index 9dec284f00..1ce2c624ca 100755 --- a/packages/contentstack-import/src/import/module-importer.ts +++ b/packages/contentstack-import/src/import/module-importer.ts @@ -51,7 +51,6 @@ class ModuleImporter { this.importConfig.backupDir = backupDir; // To support the old config this.importConfig.data = backupDir; - log(this.importConfig, `Backup directory path - '${backupDir}'`, 'info'); } await sanitizeStack(this.stackAPIClient); From 44b43c786b437130149cef7999977ae6a9e8a3c1 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Wed, 30 Aug 2023 17:38:57 +0530 Subject: [PATCH 21/28] sample config update --- packages/contentstack-export/example_config/auth_config.json | 2 +- .../contentstack-export/example_config/management_config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/contentstack-export/example_config/auth_config.json b/packages/contentstack-export/example_config/auth_config.json index 6def43821c..10a3d173c7 100644 --- a/packages/contentstack-export/example_config/auth_config.json +++ b/packages/contentstack-export/example_config/auth_config.json @@ -1,5 +1,5 @@ { - "version": 2, + "contentVersion": 2, "master_locale": { "name": "English - United States", "code": "en-us" diff --git a/packages/contentstack-export/example_config/management_config.json b/packages/contentstack-export/example_config/management_config.json index 3730fbd882..7e5efa03ea 100644 --- a/packages/contentstack-export/example_config/management_config.json +++ b/packages/contentstack-export/example_config/management_config.json @@ -1,5 +1,5 @@ { - "version": 2, + "contentVersion": 2, "master_locale": { "name": "English - United States", "code": "en-us" From 01b93dad348759e37998d024d9a08bda006060dd Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Fri, 1 Sep 2023 17:09:17 +0530 Subject: [PATCH 22/28] fix: workflow infinite loop issue --- packages/contentstack-export/src/export/modules/workflows.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/contentstack-export/src/export/modules/workflows.ts b/packages/contentstack-export/src/export/modules/workflows.ts index cdce55e359..31f4093eec 100644 --- a/packages/contentstack-export/src/export/modules/workflows.ts +++ b/packages/contentstack-export/src/export/modules/workflows.ts @@ -53,10 +53,12 @@ export default class ExportWorkFlows extends BaseClass { .fetchAll(this.qs) .then(async (data: any) => { const { items, count } = data; + //NOTE - Handle the case where old workflow api is enabled in that case getting responses as objects. + const workflowCount = count !== undefined ? count : items.length; if (items?.length) { await this.sanitizeAttribs(items); skip += this.workflowConfig.limit || 100; - if (skip >= count) { + if (skip >= workflowCount) { return; } return await this.getWorkflows(skip); From c455693ecf0844f9b8380a16373cca7ce4250cc4 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 4 Sep 2023 18:04:17 +0530 Subject: [PATCH 23/28] refactor: remove import config --- .../contentstack-export/src/export/modules/custom-roles.ts | 3 +-- .../contentstack-export/src/export/modules/environments.ts | 3 +-- .../contentstack-export/src/export/modules/extensions.ts | 3 +-- packages/contentstack-export/src/export/modules/labels.ts | 3 +-- .../src/export/modules/marketplace-apps.ts | 3 +-- packages/contentstack-export/src/export/modules/stack.ts | 3 +-- .../contentstack-export/src/export/modules/webhooks.ts | 3 +-- .../contentstack-export/src/export/modules/workflows.ts | 3 +-- .../contentstack-import/src/import/modules/custom-roles.ts | 5 ++--- .../contentstack-import/src/import/modules/environments.ts | 5 ++--- .../contentstack-import/src/import/modules/extensions.ts | 5 ++--- packages/contentstack-import/src/import/modules/labels.ts | 7 +++---- .../src/import/modules/marketplace-apps.ts | 3 +-- .../contentstack-import/src/import/modules/webhooks.ts | 5 ++--- .../contentstack-import/src/import/modules/workflows.ts | 7 +++---- 15 files changed, 23 insertions(+), 38 deletions(-) diff --git a/packages/contentstack-export/src/export/modules/custom-roles.ts b/packages/contentstack-export/src/export/modules/custom-roles.ts index 4ad66dd7f4..74397555c0 100644 --- a/packages/contentstack-export/src/export/modules/custom-roles.ts +++ b/packages/contentstack-export/src/export/modules/custom-roles.ts @@ -4,7 +4,6 @@ import forEach from 'lodash/forEach'; import values from 'lodash/values'; import { resolve as pResolve } from 'node:path'; -import config from '../../config'; import BaseClass from './base-class'; import { log, formatError, fsUtil } from '../../utils'; import { CustomRoleConfig, ModuleClassParams } from '../../types'; @@ -21,7 +20,7 @@ export default class ExportCustomRoles extends BaseClass { constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.customRoles = {}; - this.customRolesConfig = config.modules.customRoles; + this.customRolesConfig = exportConfig.modules.customRoles; this.existingRoles = { Admin: 1, Developer: 1, 'Content Manager': 1 }; this.localesMap = {}; this.sourceLocalesMap = {}; diff --git a/packages/contentstack-export/src/export/modules/environments.ts b/packages/contentstack-export/src/export/modules/environments.ts index 980b18c1ac..25e02dadac 100644 --- a/packages/contentstack-export/src/export/modules/environments.ts +++ b/packages/contentstack-export/src/export/modules/environments.ts @@ -2,7 +2,6 @@ import { resolve as pResolve } from 'node:path'; import omit from 'lodash/omit'; import isEmpty from 'lodash/isEmpty'; -import config from '../../config'; import BaseClass from './base-class'; import { log, formatError, fsUtil } from '../../utils'; import { EnvironmentConfig, ModuleClassParams } from '../../types'; @@ -19,7 +18,7 @@ export default class ExportEnvironments extends BaseClass { constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.environments = {}; - this.environmentConfig = config.modules.environments; + this.environmentConfig = exportConfig.modules.environments; this.qs = { include_count: true }; } diff --git a/packages/contentstack-export/src/export/modules/extensions.ts b/packages/contentstack-export/src/export/modules/extensions.ts index 4b1ecd514e..5eb51b2433 100644 --- a/packages/contentstack-export/src/export/modules/extensions.ts +++ b/packages/contentstack-export/src/export/modules/extensions.ts @@ -2,7 +2,6 @@ import omit from 'lodash/omit'; import isEmpty from 'lodash/isEmpty'; import { resolve as pResolve } from 'node:path'; -import config from '../../config'; import BaseClass from './base-class'; import { log, formatError, fsUtil } from '../../utils'; import { ExtensionsConfig, ModuleClassParams } from '../../types'; @@ -19,7 +18,7 @@ export default class ExportExtensions extends BaseClass { constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.extensions = {}; - this.extensionConfig = config.modules.extensions; + this.extensionConfig = exportConfig.modules.extensions; this.qs = { include_count: true }; } diff --git a/packages/contentstack-export/src/export/modules/labels.ts b/packages/contentstack-export/src/export/modules/labels.ts index ce9c320240..3bbb680f8b 100644 --- a/packages/contentstack-export/src/export/modules/labels.ts +++ b/packages/contentstack-export/src/export/modules/labels.ts @@ -2,7 +2,6 @@ import omit from 'lodash/omit'; import isEmpty from 'lodash/isEmpty'; import { resolve as pResolve } from 'node:path'; -import config from '../../config'; import BaseClass from './base-class'; import { log, formatError, fsUtil } from '../../utils'; import { LabelConfig, ModuleClassParams } from '../../types'; @@ -19,7 +18,7 @@ export default class ExportLabels extends BaseClass { constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.labels = {}; - this.labelConfig = config.modules.labels; + this.labelConfig = exportConfig.modules.labels; this.qs = { include_count: true }; } diff --git a/packages/contentstack-export/src/export/modules/marketplace-apps.ts b/packages/contentstack-export/src/export/modules/marketplace-apps.ts index 0f2fa18815..1102992e63 100644 --- a/packages/contentstack-export/src/export/modules/marketplace-apps.ts +++ b/packages/contentstack-export/src/export/modules/marketplace-apps.ts @@ -15,7 +15,6 @@ import { ContentstackClient, } from '@contentstack/cli-utilities'; -import config from '../../config'; import { log, getDeveloperHubUrl, @@ -40,7 +39,7 @@ export default class ExportMarketplaceApps extends BaseClass { constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.httpClient = new HttpClient(); - this.marketplaceAppConfig = config.modules.marketplace_apps; + this.marketplaceAppConfig = exportConfig.modules.marketplace_apps; this.listOfApps = []; this.installedApps = []; } diff --git a/packages/contentstack-export/src/export/modules/stack.ts b/packages/contentstack-export/src/export/modules/stack.ts index e05ddffcee..c5a0d7805d 100644 --- a/packages/contentstack-export/src/export/modules/stack.ts +++ b/packages/contentstack-export/src/export/modules/stack.ts @@ -2,7 +2,6 @@ import find from 'lodash/find'; import { resolve as pResolve } from 'node:path'; import { isAuthenticated, managementSDKClient } from '@contentstack/cli-utilities'; -import config from '../../config'; import BaseClass from './base-class'; import { log, formatError, fsUtil } from '../../utils'; import { StackConfig, ModuleClassParams } from '../../types'; @@ -17,7 +16,7 @@ export default class ExportStack extends BaseClass { constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); - this.stackConfig = config.modules.stack; + this.stackConfig = exportConfig.modules.stack; this.qs = { include_count: true }; this.stackFolderPath = pResolve(this.exportConfig.data, this.stackConfig.dirName); } diff --git a/packages/contentstack-export/src/export/modules/webhooks.ts b/packages/contentstack-export/src/export/modules/webhooks.ts index 77b7b99e4f..0d07b67b36 100644 --- a/packages/contentstack-export/src/export/modules/webhooks.ts +++ b/packages/contentstack-export/src/export/modules/webhooks.ts @@ -2,7 +2,6 @@ import omit from 'lodash/omit'; import isEmpty from 'lodash/isEmpty'; import { resolve as pResolve } from 'node:path'; -import config from '../../config'; import BaseClass from './base-class'; import { log, formatError, fsUtil } from '../../utils'; import { WebhookConfig, ModuleClassParams } from '../../types'; @@ -20,7 +19,7 @@ export default class ExportWebhooks extends BaseClass { constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.webhooks = {}; - this.webhookConfig = config.modules.webhooks; + this.webhookConfig = exportConfig.modules.webhooks; this.qs = { include_count: true, asc: 'updated_at' }; } diff --git a/packages/contentstack-export/src/export/modules/workflows.ts b/packages/contentstack-export/src/export/modules/workflows.ts index 31f4093eec..d757faa7b1 100644 --- a/packages/contentstack-export/src/export/modules/workflows.ts +++ b/packages/contentstack-export/src/export/modules/workflows.ts @@ -2,7 +2,6 @@ import omit from 'lodash/omit'; import isEmpty from 'lodash/isEmpty'; import { resolve as pResolve } from 'node:path'; -import config from '../../config'; import BaseClass from './base-class'; import { log, formatError, fsUtil } from '../../utils'; import { WorkflowConfig, ModuleClassParams } from '../../types'; @@ -19,7 +18,7 @@ export default class ExportWorkFlows extends BaseClass { constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.workflows = {}; - this.workflowConfig = config.modules.workflows; + this.workflowConfig = exportConfig.modules.workflows; this.qs = { include_count: true }; } diff --git a/packages/contentstack-import/src/import/modules/custom-roles.ts b/packages/contentstack-import/src/import/modules/custom-roles.ts index 05e344b699..93f0e06b58 100644 --- a/packages/contentstack-import/src/import/modules/custom-roles.ts +++ b/packages/contentstack-import/src/import/modules/custom-roles.ts @@ -2,7 +2,6 @@ import isEmpty from 'lodash/isEmpty'; import values from 'lodash/values'; import { join, resolve } from 'node:path'; -import config from '../../config'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; import { ModuleClassParams, CustomRoleConfig } from '../../types'; @@ -30,7 +29,7 @@ export default class ImportCustomRoles extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); - this.customRolesConfig = config.modules.customRoles; + this.customRolesConfig = importConfig.modules.customRoles; this.customRolesMapperPath = join(this.importConfig.backupDir, 'mapper', 'custom-roles'); this.customRolesFolderPath = join(this.importConfig.backupDir, this.customRolesConfig.dirName); this.customRolesUidMapperPath = join(this.customRolesMapperPath, 'uid-mapping.json'); @@ -155,7 +154,7 @@ export default class ImportCustomRoles extends BaseClass { entity: 'create-custom-role', includeParamOnCompletion: true, }, - concurrencyLimit: config.fetchConcurrency || 1, + concurrencyLimit: this.importConfig.fetchConcurrency || 1, }, undefined, false, diff --git a/packages/contentstack-import/src/import/modules/environments.ts b/packages/contentstack-import/src/import/modules/environments.ts index a5c7ce4446..98c933b09f 100644 --- a/packages/contentstack-import/src/import/modules/environments.ts +++ b/packages/contentstack-import/src/import/modules/environments.ts @@ -2,7 +2,6 @@ import isEmpty from 'lodash/isEmpty'; import values from 'lodash/values'; import { join, resolve } from 'node:path'; -import config from '../../config'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; import { ModuleClassParams, EnvironmentConfig } from '../../types'; @@ -21,7 +20,7 @@ export default class ImportEnvironments extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); - this.environmentsConfig = config.modules.environments; + this.environmentsConfig =importConfig.modules.environments; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'environments'); this.environmentsFolderPath = join(this.importConfig.backupDir, this.environmentsConfig.dirName); this.envUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); @@ -103,7 +102,7 @@ export default class ImportEnvironments extends BaseClass { entity: 'create-environments', includeParamOnCompletion: true, }, - concurrencyLimit: config.fetchConcurrency || 2, + concurrencyLimit: this.importConfig.fetchConcurrency || 2, }, undefined, false, diff --git a/packages/contentstack-import/src/import/modules/extensions.ts b/packages/contentstack-import/src/import/modules/extensions.ts index 6798d69b84..31b0b35f5c 100644 --- a/packages/contentstack-import/src/import/modules/extensions.ts +++ b/packages/contentstack-import/src/import/modules/extensions.ts @@ -2,7 +2,6 @@ import isEmpty from 'lodash/isEmpty'; import values from 'lodash/values'; import { join, resolve } from 'node:path'; -import config from '../../config'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; import { ModuleClassParams, Extensions } from '../../types'; @@ -21,7 +20,7 @@ export default class ImportExtensions extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); - this.extensionsConfig = config.modules.extensions; + this.extensionsConfig = importConfig.modules.extensions; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'extensions'); this.extensionsFolderPath = join(this.importConfig.backupDir, this.extensionsConfig.dirName); this.extUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); @@ -103,7 +102,7 @@ export default class ImportExtensions extends BaseClass { entity: 'create-extensions', includeParamOnCompletion: true, }, - concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, + concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, undefined, false, diff --git a/packages/contentstack-import/src/import/modules/labels.ts b/packages/contentstack-import/src/import/modules/labels.ts index e109d3714b..6bcd49283a 100644 --- a/packages/contentstack-import/src/import/modules/labels.ts +++ b/packages/contentstack-import/src/import/modules/labels.ts @@ -3,7 +3,6 @@ import values from 'lodash/values'; import omit from 'lodash/omit'; import { join, resolve } from 'node:path'; -import config from '../../config'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; import { ModuleClassParams, LabelConfig } from '../../types'; @@ -22,7 +21,7 @@ export default class Importlabels extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); - this.labelsConfig = config.modules.labels; + this.labelsConfig = importConfig.modules.labels; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'labels'); this.labelsFolderPath = join(this.importConfig.backupDir, this.labelsConfig.dirName); this.labelUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); @@ -106,7 +105,7 @@ export default class Importlabels extends BaseClass { entity: 'create-labels', includeParamOnCompletion: true, }, - concurrencyLimit: config.fetchConcurrency || 1, + concurrencyLimit: this.importConfig.fetchConcurrency || 1, }, undefined, false, @@ -158,7 +157,7 @@ export default class Importlabels extends BaseClass { entity: 'update-labels', includeParamOnCompletion: true, }, - concurrencyLimit: config.fetchConcurrency || 1, + concurrencyLimit: this.importConfig.fetchConcurrency || 1, }, undefined, false, diff --git a/packages/contentstack-import/src/import/modules/marketplace-apps.ts b/packages/contentstack-import/src/import/modules/marketplace-apps.ts index 0e9ad0cb43..e72f6a614a 100644 --- a/packages/contentstack-import/src/import/modules/marketplace-apps.ts +++ b/packages/contentstack-import/src/import/modules/marketplace-apps.ts @@ -20,7 +20,6 @@ import { ContentstackClient, } from '@contentstack/cli-utilities'; -import config from '../../config'; import BaseClass from './base-class'; import { askEncryptionKey } from '../../utils/interactive'; import { ModuleClassParams, MarketplaceAppsConfig } from '../../types'; @@ -60,7 +59,7 @@ export default class ImportMarketplaceApps extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); - this.marketPlaceAppConfig = config.modules.marketplace_apps; + this.marketPlaceAppConfig = importConfig.modules.marketplace_apps; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'marketplace_apps'); this.marketPlaceFolderPath = join(this.importConfig.backupDir, this.marketPlaceAppConfig.dirName); this.marketPlaceUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); diff --git a/packages/contentstack-import/src/import/modules/webhooks.ts b/packages/contentstack-import/src/import/modules/webhooks.ts index 10f28c5894..56c2316721 100644 --- a/packages/contentstack-import/src/import/modules/webhooks.ts +++ b/packages/contentstack-import/src/import/modules/webhooks.ts @@ -2,7 +2,6 @@ import isEmpty from 'lodash/isEmpty'; import values from 'lodash/values'; import { join, resolve } from 'node:path'; -import config from '../../config'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; import { ModuleClassParams, WebhookConfig } from '../../types'; @@ -21,7 +20,7 @@ export default class ImportWebhooks extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); - this.webhooksConfig = config.modules.webhooks; + this.webhooksConfig = importConfig.modules.webhooks; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'webhooks'); this.webhooksFolderPath = join(this.importConfig.backupDir, this.webhooksConfig.dirName); this.webhookUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); @@ -104,7 +103,7 @@ export default class ImportWebhooks extends BaseClass { entity: 'create-webhooks', includeParamOnCompletion: true, }, - concurrencyLimit: config.fetchConcurrency || 1, + concurrencyLimit: this.importConfig.fetchConcurrency || 1, }, undefined, false, diff --git a/packages/contentstack-import/src/import/modules/workflows.ts b/packages/contentstack-import/src/import/modules/workflows.ts index d403f71017..58b1a0ebcd 100644 --- a/packages/contentstack-import/src/import/modules/workflows.ts +++ b/packages/contentstack-import/src/import/modules/workflows.ts @@ -8,7 +8,6 @@ import isEmpty from 'lodash/isEmpty'; import cloneDeep from 'lodash/cloneDeep'; import findIndex from 'lodash/findIndex'; -import config from '../../config'; import BaseClass, { ApiOptions } from './base-class'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import { ModuleClassParams, WorkflowConfig } from '../../types'; @@ -28,7 +27,7 @@ export default class ImportWorkflows extends BaseClass { constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); - this.workflowsConfig = config.modules.workflows; + this.workflowsConfig = importConfig.modules.workflows; this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'workflows'); this.workflowsFolderPath = join(this.importConfig.backupDir, this.workflowsConfig.dirName); this.workflowUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); @@ -160,7 +159,7 @@ export default class ImportWorkflows extends BaseClass { entity: 'create-workflows', includeParamOnCompletion: true, }, - concurrencyLimit: config.fetchConcurrency || 1, + concurrencyLimit: this.importConfig.fetchConcurrency || 1, }, undefined, false, @@ -256,7 +255,7 @@ export default class ImportWorkflows extends BaseClass { includeParamOnCompletion: true, additionalInfo: { workflowUid: workflow.uid, stageIndex }, }, - concurrencyLimit: config.fetchConcurrency || 1, + concurrencyLimit: this.importConfig.fetchConcurrency || 1, }, undefined, false, From ea4de377b478f778c60d30d1fc316c35034ebbb6 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Tue, 5 Sep 2023 15:31:10 +0530 Subject: [PATCH 24/28] fix --- .../src/export/modules/marketplace-apps.ts | 3 +- .../src/export/modules/stack.ts | 8 +- .../src/utils/marketplace-app-helper.ts | 2 +- .../contentstack-import/src/config/index.ts | 2 +- .../src/import/modules/base-class.ts | 3 + .../src/import/modules/entries.ts | 130 ++++++++++-------- .../src/utils/entries-helper.ts | 14 +- 7 files changed, 91 insertions(+), 71 deletions(-) diff --git a/packages/contentstack-export/src/export/modules/marketplace-apps.ts b/packages/contentstack-export/src/export/modules/marketplace-apps.ts index 0f2fa18815..1102992e63 100644 --- a/packages/contentstack-export/src/export/modules/marketplace-apps.ts +++ b/packages/contentstack-export/src/export/modules/marketplace-apps.ts @@ -15,7 +15,6 @@ import { ContentstackClient, } from '@contentstack/cli-utilities'; -import config from '../../config'; import { log, getDeveloperHubUrl, @@ -40,7 +39,7 @@ export default class ExportMarketplaceApps extends BaseClass { constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.httpClient = new HttpClient(); - this.marketplaceAppConfig = config.modules.marketplace_apps; + this.marketplaceAppConfig = exportConfig.modules.marketplace_apps; this.listOfApps = []; this.installedApps = []; } diff --git a/packages/contentstack-export/src/export/modules/stack.ts b/packages/contentstack-export/src/export/modules/stack.ts index e05ddffcee..6d345e2dca 100644 --- a/packages/contentstack-export/src/export/modules/stack.ts +++ b/packages/contentstack-export/src/export/modules/stack.ts @@ -41,11 +41,9 @@ export default class ExportStack extends BaseClass { async getStack(): Promise { const tempAPIClient = await managementSDKClient({ host: this.exportConfig.host }); return await tempAPIClient - .stack({ api_key: this.exportConfig.source_stack }) - .fetch() - .catch((error: any) => { - log(this.exportConfig, `Failed to export stack. ${formatError(error)}`, 'error'); - }); + .stack({ api_key: this.exportConfig.source_stack }) + .fetch() + .catch((error: any) => {}); } async getLocales(skip: number = 0) { diff --git a/packages/contentstack-export/src/utils/marketplace-app-helper.ts b/packages/contentstack-export/src/utils/marketplace-app-helper.ts index 5ade083077..387208ab06 100644 --- a/packages/contentstack-export/src/utils/marketplace-app-helper.ts +++ b/packages/contentstack-export/src/utils/marketplace-app-helper.ts @@ -21,7 +21,7 @@ export async function getOrgUid(config: ExportConfig): Promise { .stack({ api_key: config.source_stack }) .fetch() .catch((error: any) => { - log(this.config, formatError(error), 'error'); + log(config, formatError(error), 'error'); }); return tempStackData?.org_uid; diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index faf4dd43ca..c9003f3622 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -84,7 +84,7 @@ const config: DefaultConfig = { publishAssets: true, fileName: 'assets.json', importSameStructure: true, - uploadAssetsConcurrency: 5, + uploadAssetsConcurrency: 2, displayExecutionTime: false, importFoldersConcurrency: 1, includeVersionedAssets: false, diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 6192826f3c..d70dc81ddb 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -348,6 +348,9 @@ export default abstract class BaseClass { .then(onSuccess) .catch(onReject); case 'create-entries': + if (!apiData) { + return Promise.resolve(); + } if (additionalInfo[apiData?.uid]?.isLocalized) { return apiData.update({ locale: additionalInfo.locale }).then(onSuccess).catch(onReject); } diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 51214d9f06..092da42566 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -339,36 +339,47 @@ export default class EntriesImport extends BaseClass { additionalInfo: { cTUid, locale, contentType, isMasterLocale }, } = apiOptions; - if (this.jsonRteCTs.indexOf(cTUid) > -1) { - entry = removeUidsFromJsonRteFields(entry, contentType.schema); - } - // remove entry references from json-rte fields - if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) { - entry = removeEntryRefsFromJSONRTE(entry, contentType.schema); - } - // will replace all old asset uid/urls with new ones - entry = lookupAssets( - { - content_type: contentType, - entry: entry, - }, - this.assetUidMapper, - this.assetUrlMapper, - path.join(this.entriesPath, cTUid), - this.installedExtensions, - ); - delete entry.publish_details; - // checking the entry is a localized one or not - if (!isMasterLocale && this.entriesUidMapper.hasOwnProperty(entry.uid)) { - const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry.uid]); - Object.assign(entryResponse, cloneDeep(entry), { uid: this.entriesUidMapper[entry.uid] }); - apiOptions.apiData = entryResponse; - apiOptions.additionalInfo[entryResponse.uid] = { - isLocalized: true, - }; - return apiOptions; + try { + if (this.jsonRteCTs.indexOf(cTUid) > -1) { + entry = removeUidsFromJsonRteFields(entry, contentType.schema); + } + // remove entry references from json-rte fields + if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) { + entry = removeEntryRefsFromJSONRTE(entry, contentType.schema); + } + // will replace all old asset uid/urls with new ones + entry = lookupAssets( + { + content_type: contentType, + entry: entry, + }, + this.assetUidMapper, + this.assetUrlMapper, + path.join(this.entriesPath, cTUid), + this.installedExtensions, + ); + delete entry.publish_details; + // checking the entry is a localized one or not + if (!isMasterLocale && this.entriesUidMapper.hasOwnProperty(entry.uid)) { + const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry.uid]); + Object.assign(entryResponse, cloneDeep(entry), { uid: this.entriesUidMapper[entry.uid] }); + apiOptions.apiData = entryResponse; + apiOptions.additionalInfo[entryResponse.uid] = { + isLocalized: true, + }; + return apiOptions; + } + apiOptions.apiData = entry; + } catch (error) { + log( + this.importConfig, + `${entry.title} entry of content type ${cTUid} in locale ${locale} failed to create`, + 'error', + ); + log(this.importConfig, formatError(error), 'error'); + this.failedEntries.push({ content_type: cTUid, locale, entry: { uid: entry.uid, title: entry.title } }); + apiOptions.apiData = null; } - apiOptions.apiData = entry; return apiOptions; } @@ -451,34 +462,43 @@ export default class EntriesImport extends BaseClass { apiData: entry, additionalInfo: { cTUid, locale, contentType }, } = apiOptions; + try { + const sourceEntryFilePath = entry.sourceEntryFilePath; + const sourceEntry = ((fsUtil.readFile(sourceEntryFilePath) || {}) as Record)[entry.entryOldUid]; + const newUid = this.entriesUidMapper[entry.entryOldUid]; + // Removing temp values + delete entry.sourceEntryFilePath; + delete entry.entryOldUid; + if (this.jsonRteCTs.indexOf(cTUid) > -1) { + // the entries stored in eSuccessFilePath, have the same uids as the entries from source data + entry = restoreJsonRteEntryRefs(entry, sourceEntry, contentType.schema, { + mappedAssetUids: this.assetUidMapper, + mappedAssetUrls: this.assetUrlMapper, + }); + } - const sourceEntryFilePath = entry.sourceEntryFilePath; - const sourceEntry = ((fsUtil.readFile(sourceEntryFilePath) || {}) as Record)[entry.entryOldUid]; - const newUid = this.entriesUidMapper[entry.entryOldUid]; - // Removing temp values - delete entry.sourceEntryFilePath; - delete entry.entryOldUid; - if (this.jsonRteCTs.indexOf(cTUid) > -1) { - // the entries stored in eSuccessFilePath, have the same uids as the entries from source data - entry = restoreJsonRteEntryRefs(entry, sourceEntry, contentType.schema, { - mappedAssetUids: this.assetUidMapper, - mappedAssetUrls: this.assetUrlMapper, - }); - } - - entry = lookupEntries( - { - content_type: contentType, - entry, - }, - this.entriesUidMapper, - path.join(this.entriesMapperPath, cTUid, locale), - ); + entry = lookupEntries( + { + content_type: contentType, + entry, + }, + this.entriesUidMapper, + path.join(this.entriesMapperPath, cTUid, locale), + ); - const entryResponse = this.stack.contentType(contentType.uid).entry(newUid); - Object.assign(entryResponse, cloneDeep(entry), { uid: newUid }); - delete entryResponse.publish_details; - apiOptions.apiData = entryResponse; + const entryResponse = this.stack.contentType(contentType.uid).entry(newUid); + Object.assign(entryResponse, cloneDeep(entry), { uid: newUid }); + delete entryResponse.publish_details; + apiOptions.apiData = entryResponse; + } catch (error) { + log( + this.importConfig, + `${entry.title} entry of content type ${cTUid} in locale ${locale} failed to update`, + 'error', + ); + log(this.importConfig, formatError(error), 'error'); + apiOptions.apiData = null; + } return apiOptions; } diff --git a/packages/contentstack-import/src/utils/entries-helper.ts b/packages/contentstack-import/src/utils/entries-helper.ts index 97d613308d..03b3cd68de 100644 --- a/packages/contentstack-import/src/utils/entries-helper.ts +++ b/packages/contentstack-import/src/utils/entries-helper.ts @@ -138,7 +138,7 @@ export const lookupEntries = function ( } }; - function findEntryIdsFromJsonRte(entry: any, ctSchema: any) { + function findEntryIdsFromJsonRte(entry: any, ctSchema: any = []) { for (const element of ctSchema) { switch (element.data_type) { case 'blocks': { @@ -254,7 +254,7 @@ function findUidsInNewRefFields(entry: any, uids: string[]) { export const removeUidsFromJsonRteFields = ( entry: Record, - ctSchema: Record[], + ctSchema: Record[] = [], ): Record => { for (const element of ctSchema) { switch (element.data_type) { @@ -349,7 +349,7 @@ function removeUidsFromChildren(children: Record[] | any) { } } -export const removeEntryRefsFromJSONRTE = (entry: Record, ctSchema: Record[]) => { +export const removeEntryRefsFromJSONRTE = (entry: Record, ctSchema: Record[] = []) => { for (const element of ctSchema) { switch (element.data_type) { case 'blocks': { @@ -445,7 +445,7 @@ function isEntryRef(element: any) { export const restoreJsonRteEntryRefs = ( entry: Record, sourceStackEntry: any, - ctSchema: any, + ctSchema: any = [], { mappedAssetUids, mappedAssetUrls }: any, ) => { // let mappedAssetUids = fileHelper.readFileSync(this.mappedAssetUidPath) || {}; @@ -454,7 +454,7 @@ export const restoreJsonRteEntryRefs = ( switch (element.data_type) { case 'blocks': { if (entry[element.uid]) { - if (element.multiple) { + if (element.multiple && Array.isArray(entry[element.uid])) { entry[element.uid] = entry[element.uid].map((e: any, eIndex: number) => { let key = Object.keys(e).pop(); let subBlock = element.blocks.filter((block: any) => block.uid === key).pop(); @@ -472,7 +472,7 @@ export const restoreJsonRteEntryRefs = ( case 'global_field': case 'group': { if (entry[element.uid]) { - if (element.multiple) { + if (element.multiple && Array.isArray(entry[element.uid])) { entry[element.uid] = entry[element.uid].map((e: any, eIndex: number) => { let sourceStackElement = sourceStackEntry[element.uid][eIndex]; e = restoreJsonRteEntryRefs(e, sourceStackElement, element.schema, { mappedAssetUids, mappedAssetUrls }); @@ -490,7 +490,7 @@ export const restoreJsonRteEntryRefs = ( } case 'json': { if (entry[element.uid] && element.field_metadata.rich_text_type) { - if (element.multiple) { + if (element.multiple && Array.isArray(entry[element.uid])) { entry[element.uid] = entry[element.uid].map((field: any, index: number) => { // i am facing a Maximum call stack exceeded issue, // probably because of this loop operation From c1573f734e632b0d7ad208d835f714c2d8f23c88 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Wed, 6 Sep 2023 11:52:30 +0530 Subject: [PATCH 25/28] fixed conflict --- packages/contentstack-branches/README.md | 2 +- .../src/import/modules/entries.ts | 12 +++++------- packages/contentstack/README.md | 2 +- pnpm-lock.yaml | 10 ++++++++-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/contentstack-branches/README.md b/packages/contentstack-branches/README.md index d961644214..6d675b4cf7 100755 --- a/packages/contentstack-branches/README.md +++ b/packages/contentstack-branches/README.md @@ -37,7 +37,7 @@ $ npm install -g @contentstack/cli-cm-branches $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-branches/1.0.11 darwin-arm64 node-v18.11.0 +@contentstack/cli-cm-branches/1.0.12 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index ebf5e04c57..5523be3061 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -577,20 +577,18 @@ export default class EntriesImport extends BaseClass { return; } for (let cTUid of cTsWithFieldRules) { - const cTs: Record[] = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record[]; + const cTs: Record[] = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record< + string, + unknown + >[]; const contentType: any = find(cTs, { uid: cTUid }); if (contentType.field_rules) { - const fieldDatatypeMap: any = {}; + const fieldDatatypeMap: { [key: string]: string } = {}; for (let i = 0; i < contentType.schema.length; i++) { const field = contentType.schema[i].uid; fieldDatatypeMap[field] = contentType.schema[i].data_type; } let fieldRuleLength = contentType.field_rules.length; - const fieldDatatypeMap: {[key: string]: string} = {}; - for (let i = 0; i < contentType.schema.length; i++) { - const field = contentType.schema[i].uid; - fieldDatatypeMap[field] = contentType.schema[i].data_type; - } for (let k = 0; k < fieldRuleLength; k++) { let fieldRuleConditionLength = contentType.field_rules[k].conditions.length; for (let i = 0; i < fieldRuleConditionLength; i++) { diff --git a/packages/contentstack/README.md b/packages/contentstack/README.md index aa7be6a19d..1092baa6ba 100644 --- a/packages/contentstack/README.md +++ b/packages/contentstack/README.md @@ -18,7 +18,7 @@ $ npm install -g @contentstack/cli $ csdx COMMAND running command... $ csdx (--version|-v) -@contentstack/cli/1.8.2 darwin-arm64 node-v18.11.0 +@contentstack/cli/1.8.3 darwin-arm64 node-v18.11.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3b7b43bd39..396ed75661 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -634,7 +634,7 @@ importers: globby: ^10.0.2 inquirer: 8.2.4 inquirer-checkbox-plus-prompt: 1.0.1 - mkdirp: ^1.0.4 + mkdirp: ^3.0.1 mocha: ^10.0.0 nyc: ^15.1.0 oclif: ^3.8.1 @@ -645,7 +645,7 @@ importers: fast-csv: 4.3.6 inquirer: 8.2.4 inquirer-checkbox-plus-prompt: 1.0.1 - mkdirp: 1.0.4 + mkdirp: 3.0.1 devDependencies: '@oclif/test': 2.3.31 chai: 4.3.7 @@ -9214,6 +9214,12 @@ packages: engines: {node: '>=10'} hasBin: true + /mkdirp/3.0.1: + resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==} + engines: {node: '>=10'} + hasBin: true + dev: false + /mkdirp2/1.0.5: resolution: {integrity: sha512-xOE9xbICroUDmG1ye2h4bZ8WBie9EGmACaco8K8cx6RlkJJrxGIqjGqztAI+NMhexXBcdGbSEzI6N3EJPevxZw==} dev: true From 58b9eb0ff5f14f72d987d37621df411e118e63c9 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Wed, 6 Sep 2023 17:24:00 +0530 Subject: [PATCH 26/28] message upted --- .../contentstack-import/src/import/modules/base-class.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index d70dc81ddb..fa99462ead 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -206,11 +206,7 @@ export default abstract class BaseClass { // info: Batch No. 20 of import assets is complete if (currentIndexer) batchMsg += `Current chunk processing is (${currentIndexer}/${indexerCount})`; - log( - this.importConfig, - `Batch No. (${batchNo}/${totelBatches}) of ${processName} is complete. ${batchMsg}`, - 'success', - ); + log(this.importConfig, `Batch No. (${batchNo}/${totelBatches}) of ${processName} is complete`, 'success'); } if (this.importConfig.modules.assets.displayExecutionTime) { From df9c3d7fc8fb6ee3ee13532190ae5b85c1612a07 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Tue, 12 Sep 2023 17:31:06 +0530 Subject: [PATCH 27/28] fix: added fix for mapper file issue --- .../src/import/modules/entries.ts | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 5523be3061..fa83c02d06 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -284,14 +284,27 @@ export default class EntriesImport extends BaseClass { const contentType = find(this.cTs, { uid: cTUid }); const onSuccess = ({ response, apiData: entry, additionalInfo }: any) => { - log(this.importConfig, `Created entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info'); - if (!isMasterLocale && !additionalInfo[entry.uid]?.isLocalized) { - this.autoCreatedEntries.push({ cTUid, locale, entryUid: response.uid }); + if (additionalInfo[entry.uid]?.isLocalized) { + let oldUid = additionalInfo[entry.uid].entryOldUid; + log(this.importConfig, `Localized entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info'); + entry.uid = oldUid; + entry.entryOldUid = oldUid; + entry.sourceEntryFilePath = path.join(basePath, additionalInfo.entryFileName); // stores source file path temporarily + entriesCreateFileHelper.writeIntoFile({ [oldUid]: entry } as any, { mapKeyVal: true }); + } else { + log(this.importConfig, `Created entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info'); + // This is for creating localized entries that do not have a counterpart in master locale. + // For example : To create entry1 in fr-fr, where en-us is the master locale + // entry1 will get created in en-us first, then fr-fr version will be created + // thus entry1 has to be removed from en-us at the end. + if (!isMasterLocale && !additionalInfo[entry.uid]?.isLocalized) { + this.autoCreatedEntries.push({ cTUid, locale, entryUid: response.uid }); + } + this.entriesUidMapper[entry.uid] = response.uid; + entry.sourceEntryFilePath = path.join(basePath, additionalInfo.entryFileName); // stores source file path temporarily + entry.entryOldUid = entry.uid; // stores old uid temporarily + entriesCreateFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true }); } - this.entriesUidMapper[entry.uid] = response.uid; - entry.sourceEntryFilePath = path.join(basePath, additionalInfo.entryFileName); // stores source file path temporarily - entry.entryOldUid = entry.uid; // stores old uid temporarily - entriesCreateFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true }); }; const onReject = ({ error, apiData: { uid, title } }: any) => { log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to create`, 'error'); @@ -366,6 +379,7 @@ export default class EntriesImport extends BaseClass { apiOptions.apiData = entryResponse; apiOptions.additionalInfo[entryResponse.uid] = { isLocalized: true, + entryOldUid: entry.uid }; return apiOptions; } From 8b933fbcdc56125869718acc3e361ed812ea2781 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Wed, 20 Sep 2023 15:20:24 +0530 Subject: [PATCH 28/28] update graphql version --- package-lock.json | 16 +++++++------- packages/contentstack-launch/package.json | 4 ++-- pnpm-lock.yaml | 26 +++++++++++------------ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2e51e53be0..dbcde7186e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8458,9 +8458,9 @@ "dev": true }, "node_modules/graphql": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.7.1.tgz", - "integrity": "sha512-DRYR9tf+UGU0KOsMcKAlXeFfX89UiiIZ0dRU3mR0yJfu6OjZqUcp68NnFLnqQU5RexygFoDy1EW+ccOYcPfmHg==", + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -24341,7 +24341,7 @@ "esm": "^3.2.25", "express": "^4.18.2", "form-data": "^4.0.0", - "graphql": "^16.6.0", + "graphql": "^16.8.1", "ini": "^3.0.1", "lodash": "^4.17.21", "open": "^8.4.2", @@ -27640,7 +27640,7 @@ "esm": "^3.2.25", "express": "^4.18.2", "form-data": "^4.0.0", - "graphql": "^16.6.0", + "graphql": "^16.8.1", "ini": "^3.0.1", "lodash": "^4.17.21", "oclif": "^3", @@ -34774,9 +34774,9 @@ "dev": true }, "graphql": { - "version": "16.7.1", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.7.1.tgz", - "integrity": "sha512-DRYR9tf+UGU0KOsMcKAlXeFfX89UiiIZ0dRU3mR0yJfu6OjZqUcp68NnFLnqQU5RexygFoDy1EW+ccOYcPfmHg==" + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==" }, "graphql-tag": { "version": "2.12.6", diff --git a/packages/contentstack-launch/package.json b/packages/contentstack-launch/package.json index a14dc55008..9d31b9f1ce 100755 --- a/packages/contentstack-launch/package.json +++ b/packages/contentstack-launch/package.json @@ -32,7 +32,7 @@ "esm": "^3.2.25", "express": "^4.18.2", "form-data": "^4.0.0", - "graphql": "^16.6.0", + "graphql": "^16.8.1", "ini": "^3.0.1", "lodash": "^4.17.21", "open": "^8.4.2", @@ -105,4 +105,4 @@ "launch:deployments": "LNCH-DPLMNT" } } -} +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index df509b21ec..72ff4186cc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -760,7 +760,7 @@ importers: esm: ^3.2.25 express: ^4.18.2 form-data: ^4.0.0 - graphql: ^16.6.0 + graphql: ^16.8.1 ini: ^3.0.1 lodash: ^4.17.21 oclif: ^3 @@ -771,7 +771,7 @@ importers: typescript: ^4.9.5 winston: ^3.8.2 dependencies: - '@apollo/client': 3.7.17_graphql@16.7.1 + '@apollo/client': 3.7.17_graphql@16.8.1 '@contentstack/cli-command': link:../contentstack-command '@contentstack/cli-utilities': link:../contentstack-utilities '@oclif/core': 2.9.4_npeoqfxwgl5njwqojmpesqhar4 @@ -786,7 +786,7 @@ importers: esm: 3.2.25 express: 4.18.2 form-data: 4.0.0 - graphql: 16.7.1 + graphql: 16.8.1 ini: 3.0.1 lodash: 4.17.21 open: 8.4.2 @@ -1064,7 +1064,7 @@ packages: '@jridgewell/trace-mapping': 0.3.18 dev: true - /@apollo/client/3.7.17_graphql@16.7.1: + /@apollo/client/3.7.17_graphql@16.8.1: resolution: {integrity: sha512-0EErSHEtKPNl5wgWikHJbKFAzJ/k11O0WO2QyqZSHpdxdAnw7UWHY4YiLbHCFG7lhrD+NTQ3Z/H9Jn4rcikoJA==} peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 @@ -1082,12 +1082,12 @@ packages: subscriptions-transport-ws: optional: true dependencies: - '@graphql-typed-document-node/core': 3.2.0_graphql@16.7.1 + '@graphql-typed-document-node/core': 3.2.0_graphql@16.8.1 '@wry/context': 0.7.3 '@wry/equality': 0.5.6 '@wry/trie': 0.4.3 - graphql: 16.7.1 - graphql-tag: 2.12.6_graphql@16.7.1 + graphql: 16.8.1 + graphql-tag: 2.12.6_graphql@16.8.1 hoist-non-react-statics: 3.3.2 optimism: 0.16.2 prop-types: 15.8.1 @@ -1607,12 +1607,12 @@ packages: resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} dev: true - /@graphql-typed-document-node/core/3.2.0_graphql@16.7.1: + /@graphql-typed-document-node/core/3.2.0_graphql@16.8.1: resolution: {integrity: sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==} peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - graphql: 16.7.1 + graphql: 16.8.1 dev: false /@humanwhocodes/config-array/0.11.10: @@ -6946,18 +6946,18 @@ packages: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} dev: true - /graphql-tag/2.12.6_graphql@16.7.1: + /graphql-tag/2.12.6_graphql@16.8.1: resolution: {integrity: sha512-FdSNcu2QQcWnM2VNvSCCDCVS5PpPqpzgFT8+GXzqJuoDd0CBncxCY278u4mhRO7tMgo2JjgJA5aZ+nWSQ/Z+xg==} engines: {node: '>=10'} peerDependencies: graphql: ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 dependencies: - graphql: 16.7.1 + graphql: 16.8.1 tslib: 2.6.0 dev: false - /graphql/16.7.1: - resolution: {integrity: sha512-DRYR9tf+UGU0KOsMcKAlXeFfX89UiiIZ0dRU3mR0yJfu6OjZqUcp68NnFLnqQU5RexygFoDy1EW+ccOYcPfmHg==} + /graphql/16.8.1: + resolution: {integrity: sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} dev: false