From 9f02cb5a8fc7addfc4dd8ac0e15edf76728727b2 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 25 Sep 2023 16:44:27 +0530 Subject: [PATCH 1/4] feat: import taxonomies --- .../contentstack-import/src/config/index.ts | 9 + .../src/import/modules/base-class.ts | 18 +- .../src/import/modules/taxonomies.ts | 247 ++++++++++++++++++ .../src/types/default-config.ts | 10 + .../contentstack-import/src/types/index.ts | 17 +- 5 files changed, 299 insertions(+), 2 deletions(-) create mode 100644 packages/contentstack-import/src/import/modules/taxonomies.ts diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index c9003f3622..3a53ae0baf 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -28,6 +28,7 @@ const config: DefaultConfig = { 'assets', 'extensions', 'marketplace-apps', + 'taxonomies', 'global-fields', 'content-types', 'custom-roles', @@ -142,6 +143,14 @@ const config: DefaultConfig = { dirName: 'marketplace_apps', fileName: 'marketplace_apps.json', }, + taxonomies: { + dirName: 'taxonomies', + fileName: 'taxonomies.json' + }, + terms: { + dirName: 'terms', + fileName: 'terms.json' + }, }, languagesCode: [ 'af-za', diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index fa99462ead..e47f8790ab 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -18,6 +18,7 @@ import { LabelData } from '@contentstack/management/types/stack/label'; import { WebhookData } from '@contentstack/management/types/stack/webhook'; import { WorkflowData } from '@contentstack/management/types/stack/workflow'; import { RoleData } from '@contentstack/management/types/stack/role'; +import { HttpClient } from '@contentstack/cli-utilities'; import { log } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; @@ -47,7 +48,9 @@ export type ApiModuleType = | 'create-entries' | 'update-entries' | 'publish-entries' - | 'delete-entries'; + | 'delete-entries' + | 'create-taxonomies' + | 'create-terms'; export type ApiOptions = { uid?: string; @@ -381,6 +384,19 @@ export default abstract class BaseClass { .delete({ locale: additionalInfo.locale }) .then(onSuccess) .catch(onReject); + case 'create-taxonomies': + return new HttpClient() + .headers(additionalInfo.headers) + .post(additionalInfo.url, { taxonomy: apiData }) + .then(onSuccess) + .catch(onReject); + case 'create-terms': + const url = `${additionalInfo.baseUrl}/${apiData.taxonomy_uid}/terms`; + return new HttpClient() + .headers(additionalInfo.headers) + .post(url, { term: apiData }) + .then(onSuccess) + .catch(onReject); default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts new file mode 100644 index 0000000000..d0b5b9e820 --- /dev/null +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -0,0 +1,247 @@ +import isEmpty from 'lodash/isEmpty'; +import values from 'lodash/values'; +import keys from 'lodash/keys'; +import flatten from 'lodash/flatten'; +import { join, resolve as pResolve } from 'node:path'; +import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; + +import config from '../../config'; +import { log, formatError, fsUtil, fileHelper } from '../../utils'; +import BaseClass, { ApiOptions } from './base-class'; +import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; + +//NOTE: Temp types need to remove once sdk available +type TaxonomyPayload = { + baseUrl: string; + url: string; + mgToken: string; + reqPayload: Record; + headers: Record; +}; + +export default class ImportTaxonomies extends BaseClass { + private taxonomiesMapperDirPath: string; + private termsMapperDirPath: string; + private taxonomiesFolderPath: string; + private termsFolderPath: string; + private taxSuccessPath: string; + private taxFailsPath: string; + private taxonomiesConfig: TaxonomiesConfig; + private termsConfig: TermsConfig; + private taxonomies: Record; + private taxonomiesSuccess: Record = {}; + private taxonomiesFailed: Record = {}; + private taxonomyPayload: TaxonomyPayload; + + constructor({ importConfig, stackAPIClient }: ModuleClassParams) { + super({ importConfig, stackAPIClient }); + this.taxonomiesConfig = importConfig.modules.taxonomies; + this.termsConfig = importConfig.modules.terms; + this.taxonomiesMapperDirPath = join(importConfig.backupDir, 'mapper', 'taxonomies'); + this.termsMapperDirPath = join(this.taxonomiesMapperDirPath, 'terms'); + this.taxonomiesFolderPath = join(importConfig.backupDir, this.taxonomiesConfig.dirName); + this.termsFolderPath = join(this.taxonomiesFolderPath, this.termsConfig.dirName); + this.taxSuccessPath = join(this.taxonomiesMapperDirPath, 'success.json'); + this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json'); + this.taxonomyPayload = { + baseUrl: '', + url: '', + mgToken: importConfig.management_token, + reqPayload: {}, + headers: { + 'Content-Type': 'application/json', + api_key: importConfig.target_stack, + }, + }; + } + + /** + * @method start + * @returns {Promise} Promise + */ + async start(): Promise { + log(this.importConfig, 'Migrating taxonomies', 'info'); + + //Step1 check folder exists or not + if (fileHelper.fileExistsSync(this.taxonomiesFolderPath)) { + this.taxonomies = fsUtil.readFile(join(this.taxonomiesFolderPath, 'taxonomies.json'), true) as Record< + string, + unknown + >; + } else { + log(this.importConfig, `No such file or directory - '${this.taxonomiesFolderPath}'`, 'error'); + return; + } + + //NOTE - Temp code for api request + const { cma } = configHandler.get('region') || {}; + this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; + this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; + if (this.taxonomyPayload?.mgToken) this.taxonomyPayload.headers['authorization'] = this.taxonomyPayload.mgToken; + else this.taxonomyPayload.headers['authtoken'] = configHandler.get('authtoken'); + + //Step 2 create taxonomies & terms mapper directory + await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); + await fsUtil.makeDirectory(this.termsMapperDirPath); + + await this.importTaxonomies(); + // create terms related to respective taxonomy + if (!fileHelper.fileExistsSync(this.termsFolderPath)) { + log(this.importConfig, `No such file or directory - '${this.taxonomiesFolderPath}'`, 'error'); + return; + } + await this.importTerms(); + + if (this.taxonomiesSuccess !== undefined && !isEmpty(this.taxonomiesSuccess)) { + fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesSuccess); + } + + if (this.taxonomiesFailed !== undefined &&!isEmpty(this.taxonomiesFailed)) { + fsUtil.writeFile(this.taxFailsPath, this.taxonomiesFailed); + } + + log(this.importConfig, 'Taxonomies have been imported successfully!', 'success'); + } + + async importTaxonomies(): Promise { + if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { + log(this.importConfig, 'No Taxonomies Found', 'info'); + return; + } + + const apiContent = values(this.taxonomies); + + const onSuccess = ({ + response: { data, status } = { data: null, status: null }, + apiData: { uid, name } = { uid: null, name: '' }, + }: any) => { + //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready + if ([200, 201, 202].includes(status)) { + this.taxonomiesSuccess[uid] = data; + log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); + } else { + let errorMsg; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`; + log(this.importConfig, `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error'); + } + }; + + const onReject = ({ error, apiData }: any) => { + const err = error?.message ? JSON.parse(error.message) : error; + const { name } = apiData; + if (err?.errors?.name) { + log(this.importConfig, `Taxonomy '${name}' already exists`, 'info'); + } else { + this.taxonomiesFailed[apiData.uid] = apiData; + log(this.importConfig, `Taxonomy '${name}' failed to be import ${formatError(error)}`, 'error'); + log(this.importConfig, error, 'error'); + } + }; + + await this.makeConcurrentCall( + { + apiContent, + processName: 'import taxonomies', + apiParams: { + serializeData: this.serializeTaxonomy.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'create-taxonomies', + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, + }, + concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, + }, + undefined, + false, + ); + } + + /** + * @method serializeTaxonomy + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { + const { apiData: taxonomy } = apiOptions; + if (this.taxonomiesSuccess.hasOwnProperty(taxonomy.uid)) { + log(this.importConfig, `Taxonomy '${taxonomy.title}' already exists. Skipping it to avoid duplicates!`, 'info'); + apiOptions.entity = undefined; + } else { + apiOptions.apiData = taxonomy; + } + apiOptions.apiData = taxonomy; + return apiOptions; + } + + async importTerms(): Promise { + if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { + return; + } + + const listOfTaxonomyUIDs = keys(this.taxonomies); + + const onSuccess = ({ + response, + apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: '' }, + }: any) => { + this.taxonomiesSuccess[uid] = response; + log(this.importConfig, `Term '${name}' imported successfully`, 'success'); + }; + + const onReject = ({ error, apiData }: any) => { + const err = error?.message ? JSON.parse(error.message) : error; + const { name } = apiData; + if (err?.errors?.name) { + log(this.importConfig, `Term '${name}' already exists`, 'info'); + } else { + this.taxonomiesSuccess[apiData.uid] = apiData; + log(this.importConfig, `Term '${name}' failed to be import ${formatError(error)}`, 'error'); + log(this.importConfig, error, 'error'); + } + }; + + for (const taxUID of listOfTaxonomyUIDs) { + const terms = fsUtil.readFile( + join(this.termsFolderPath, `${taxUID}-${this.termsConfig.fileName}`), + true, + ) as Record; + const dirPath = pResolve(this.termsMapperDirPath, `${taxUID}-terms`); + if (!fileHelper.fileExistsSync(dirPath)) { + await fsUtil.makeDirectory(dirPath); + } + //success and fail path for particular taxonomy uid + const apiContent = values(terms); + await this.makeConcurrentCall( + { + apiContent, + processName: 'import terms', + apiParams: { + serializeData: this.serializeTerms.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'create-terms', + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, + }, + concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, + }, + undefined, + false, + ); + } + } + + /** + * @method serializeTerms + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeTerms(apiOptions: ApiOptions): ApiOptions { + const { apiData: term } = apiOptions; + apiOptions.apiData = term; + return apiOptions; + } +} diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index c41b949fdb..27b0af0859 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -113,6 +113,16 @@ export default interface DefaultConfig { fileName: string; requiredKeys: string[]; }; + taxonomies: { + dirName: string; + fileName: string; + dependencies?: Modules[]; + }; + terms: { + dirName: string; + fileName: string; + dependencies?: Modules[]; + }; }; languagesCode: string[]; apis: { diff --git a/packages/contentstack-import/src/types/index.ts b/packages/contentstack-import/src/types/index.ts index 104328215f..53a6b1f61e 100644 --- a/packages/contentstack-import/src/types/index.ts +++ b/packages/contentstack-import/src/types/index.ts @@ -40,7 +40,8 @@ export type Modules = | 'custom-roles' | 'workflows' | 'labels' - | 'marketplace-apps'; + | 'marketplace-apps' + | 'taxonomies'; export type ModuleClassParams = { stackAPIClient: ReturnType; @@ -88,5 +89,19 @@ export interface CustomRoleConfig{ customRolesLocalesFileName: string; } +export interface TaxonomiesConfig{ + dirName: string; + fileName: string; + dependencies?: Modules[]; + limit?: number; +} + +export interface TermsConfig{ + dirName: string; + fileName: string; + dependencies?: Modules[]; + limit?: number; +} + export { default as DefaultConfig } from './default-config'; export { default as ImportConfig } from './import-config'; From 1f1b7471d39e8f92b7a728694d0632696621fd2d Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 26 Sep 2023 16:51:06 +0530 Subject: [PATCH 2/4] feat: import terms --- .../src/commands/cm/export-to-csv.js | 2 +- .../src/export/modules/taxonomies.ts | 3 +- .../contentstack-import/src/config/index.ts | 2 +- .../src/import/modules/taxonomies.ts | 167 ++++++++++++------ .../src/utils/asset-helper.ts | 2 +- 5 files changed, 119 insertions(+), 57 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 26cbe7fe2e..1898bbeacc 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -277,7 +277,7 @@ class ExportToCsvCommand extends Command { throw new Error(branchExists.errorMessage); } stack.branch_uid = branchUid; - stackAPIClient = getStackClient(managementAPIClient, stack); + stackAPIClient = this.getStackClient(managementAPIClient, stack); } catch (error) { if (error?.message || error?.errorMessage) { cliux.error(util.formatError(error)); diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index fa463a880f..6e50298a50 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -120,7 +120,7 @@ export default class ExportTaxonomies extends BaseClass { log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'`, 'info'); } else { fsUtil.writeFile(pResolve(this.termsFolderPath, `${taxonomyUID}-${this.termsConfig.fileName}`), this.terms); - log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' were successfully exported.`, 'success'); + log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' were exported successfully.`, 'success'); } } log(this.exportConfig, `All the terms have been exported successfully!`, 'success'); @@ -172,6 +172,7 @@ export default class ExportTaxonomies extends BaseClass { include_count: true, skip: 0, limit: this.taxonomiesConfig.limit || 100, + depth: 0 // include all the terms if set to 0 }; if (skip >= 0) params['skip'] = skip; diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 3a53ae0baf..315565ab28 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -26,9 +26,9 @@ const config: DefaultConfig = { 'locales', 'environments', 'assets', + 'taxonomies', 'extensions', 'marketplace-apps', - 'taxonomies', 'global-fields', 'content-types', 'custom-roles', diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index d0b5b9e820..551ff7822f 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -2,10 +2,9 @@ import isEmpty from 'lodash/isEmpty'; import values from 'lodash/values'; import keys from 'lodash/keys'; import flatten from 'lodash/flatten'; -import { join, resolve as pResolve } from 'node:path'; -import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; +import { join } from 'node:path'; +import { configHandler } from '@contentstack/cli-utilities'; -import config from '../../config'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; @@ -21,17 +20,23 @@ type TaxonomyPayload = { export default class ImportTaxonomies extends BaseClass { private taxonomiesMapperDirPath: string; - private termsMapperDirPath: string; private taxonomiesFolderPath: string; - private termsFolderPath: string; private taxSuccessPath: string; private taxFailsPath: string; private taxonomiesConfig: TaxonomiesConfig; - private termsConfig: TermsConfig; private taxonomies: Record; - private taxonomiesSuccess: Record = {}; - private taxonomiesFailed: Record = {}; + private termsFolderPath: string; + private termsMapperDirPath: string; + private termsConfig: TermsConfig; + private termsSuccessPath: string; + private termsFailsPath: string; private taxonomyPayload: TaxonomyPayload; + public taxonomiesSuccess: Record = {}; + public taxonomiesFailed: Record = {}; + public termsSuccess: Record> = {}; + public termsFailed: Record> = {}; + public terms: Record = {}; + public taxonomyUIDs: string[] = []; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -43,6 +48,8 @@ export default class ImportTaxonomies extends BaseClass { this.termsFolderPath = join(this.taxonomiesFolderPath, this.termsConfig.dirName); this.taxSuccessPath = join(this.taxonomiesMapperDirPath, 'success.json'); this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json'); + this.termsSuccessPath = join(this.termsMapperDirPath, 'success.json'); + this.termsFailsPath = join(this.termsMapperDirPath, 'fails.json'); this.taxonomyPayload = { baseUrl: '', url: '', @@ -84,25 +91,27 @@ export default class ImportTaxonomies extends BaseClass { await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); await fsUtil.makeDirectory(this.termsMapperDirPath); + //Step 3 import taxonomy and create success & failure file await this.importTaxonomies(); - // create terms related to respective taxonomy + this.createTaxonomySuccessAndFailedFile(); + if (!fileHelper.fileExistsSync(this.termsFolderPath)) { - log(this.importConfig, `No such file or directory - '${this.taxonomiesFolderPath}'`, 'error'); + log(this.importConfig, `No such file or directory - '${this.termsFolderPath}'`, 'error'); return; } + //Step 4 import terms and create success & failure file await this.importTerms(); - - if (this.taxonomiesSuccess !== undefined && !isEmpty(this.taxonomiesSuccess)) { - fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesSuccess); - } - - if (this.taxonomiesFailed !== undefined &&!isEmpty(this.taxonomiesFailed)) { - fsUtil.writeFile(this.taxFailsPath, this.taxonomiesFailed); - } + this.createTermSuccessAndFailedFile(); log(this.importConfig, 'Taxonomies have been imported successfully!', 'success'); } + /** + * create taxonomy and enter success & failure related data into taxonomies mapper file + * @method importTaxonomies + * @async + * @returns {Promise} Promise + */ async importTaxonomies(): Promise { if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { log(this.importConfig, 'No Taxonomies Found', 'info'); @@ -110,6 +119,7 @@ export default class ImportTaxonomies extends BaseClass { } const apiContent = values(this.taxonomies); + this.taxonomyUIDs = keys(this.taxonomies); const onSuccess = ({ response: { data, status } = { data: null, status: null }, @@ -117,12 +127,16 @@ export default class ImportTaxonomies extends BaseClass { }: any) => { //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready if ([200, 201, 202].includes(status)) { - this.taxonomiesSuccess[uid] = data; + const { taxonomy } = data; + this.taxonomiesSuccess[taxonomy.uid] = taxonomy; log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); } else { let errorMsg; if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; else errorMsg = data?.error_message; + if (errorMsg === undefined) { + errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); + } this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`; log(this.importConfig, `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error'); } @@ -152,7 +166,7 @@ export default class ImportTaxonomies extends BaseClass { includeParamOnCompletion: true, additionalInfo: this.taxonomyPayload, }, - concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, + concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, undefined, false, @@ -176,61 +190,94 @@ export default class ImportTaxonomies extends BaseClass { return apiOptions; } + /** + * create taxonomies success and fail in (mapper/taxonomies) + * @method createTaxonomySuccessAndFailedFile + */ + createTaxonomySuccessAndFailedFile() { + if (this.taxonomiesSuccess !== undefined && !isEmpty(this.taxonomiesSuccess)) { + fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesSuccess); + } + + if (this.taxonomiesFailed !== undefined && !isEmpty(this.taxonomiesFailed)) { + fsUtil.writeFile(this.taxFailsPath, this.taxonomiesFailed); + } + } + + /** + * create terms and enter success & failure related data into terms mapper file + * @method importTerms + * @async + * @returns {Promise} Promise + */ async importTerms(): Promise { - if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { + if (!this.taxonomyUIDs?.length) { return; } - const listOfTaxonomyUIDs = keys(this.taxonomies); - const onSuccess = ({ - response, - apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: '' }, + response: { data, status } = { data: null, status: null }, + apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: null }, }: any) => { - this.taxonomiesSuccess[uid] = response; - log(this.importConfig, `Term '${name}' imported successfully`, 'success'); + //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready + if ([200, 201, 202].includes(status)) { + if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; + const { term } = data; + this.termsSuccess[taxonomy_uid][term.uid] = term; + log(this.importConfig, `Term '${name}' imported successfully`, 'success'); + } else { + if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; + let errorMsg; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + if (errorMsg === undefined) { + errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); + } + this.termsFailed[taxonomy_uid][uid] = `Terms '${name}' failed to be import. ${JSON.stringify(errorMsg)}`; + log(this.importConfig, `Terms '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error'); + } }; const onReject = ({ error, apiData }: any) => { + const { uid, taxonomy_uid, name } = apiData; + if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; const err = error?.message ? JSON.parse(error.message) : error; - const { name } = apiData; if (err?.errors?.name) { log(this.importConfig, `Term '${name}' already exists`, 'info'); } else { - this.taxonomiesSuccess[apiData.uid] = apiData; + this.termsFailed[taxonomy_uid][apiData.uid] = apiData; log(this.importConfig, `Term '${name}' failed to be import ${formatError(error)}`, 'error'); log(this.importConfig, error, 'error'); } }; - for (const taxUID of listOfTaxonomyUIDs) { - const terms = fsUtil.readFile( + for (const taxUID of this.taxonomyUIDs) { + //read terms from respective taxonomy + this.terms = fsUtil.readFile( join(this.termsFolderPath, `${taxUID}-${this.termsConfig.fileName}`), true, ) as Record; - const dirPath = pResolve(this.termsMapperDirPath, `${taxUID}-terms`); - if (!fileHelper.fileExistsSync(dirPath)) { - await fsUtil.makeDirectory(dirPath); - } - //success and fail path for particular taxonomy uid - const apiContent = values(terms); - await this.makeConcurrentCall( - { - apiContent, - processName: 'import terms', - apiParams: { - serializeData: this.serializeTerms.bind(this), - reject: onReject, - resolve: onSuccess, - entity: 'create-terms', - includeParamOnCompletion: true, - additionalInfo: this.taxonomyPayload, + + if (this.terms !== undefined && !isEmpty(this.terms)) { + const apiContent = values(this.terms); + await this.makeConcurrentCall( + { + apiContent, + processName: 'import terms', + apiParams: { + serializeData: this.serializeTerms.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'create-terms', + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, + }, + concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, - concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, - }, - undefined, - false, - ); + undefined, + false, + ); + } } } @@ -244,4 +291,18 @@ export default class ImportTaxonomies extends BaseClass { apiOptions.apiData = term; return apiOptions; } + + /** + * create terms success and fail in (mapper/taxonomies/terms) + * @method createTermSuccessAndFailedFile + */ + createTermSuccessAndFailedFile() { + if (this.termsSuccess !== undefined && !isEmpty(this.termsSuccess)) { + fsUtil.writeFile(this.termsSuccessPath, this.termsSuccess); + } + + if (this.termsFailed !== undefined && !isEmpty(this.termsFailed)) { + fsUtil.writeFile(this.termsFailsPath, this.termsFailed); + } + } } diff --git a/packages/contentstack-import/src/utils/asset-helper.ts b/packages/contentstack-import/src/utils/asset-helper.ts index 7e7cc9e257..ceaead0bed 100644 --- a/packages/contentstack-import/src/utils/asset-helper.ts +++ b/packages/contentstack-import/src/utils/asset-helper.ts @@ -2,7 +2,7 @@ import Bluebird from 'bluebird'; import * as url from 'url'; import * as path from 'path'; import { ContentstackClient, managementSDKClient } from '@contentstack/cli-utilities'; -import { ImportConfig } from 'src/types'; +import { ImportConfig } from '../types'; const debug = require('debug')('util:requests'); let _ = require('lodash'); let { marked } = require('marked'); From 0fc7e724c034f296f951be16c559d2c041fb0e29 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 26 Sep 2023 17:13:04 +0530 Subject: [PATCH 3/4] refactor: taxonomy export & import --- .../contentstack-export/src/export/modules/taxonomies.ts | 2 +- .../contentstack-import/src/import/modules/taxonomies.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index 6e50298a50..09aa0db139 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -1,7 +1,7 @@ import omit from 'lodash/omit'; +import keys from 'lodash/keys'; import isEmpty from 'lodash/isEmpty'; import flatten from 'lodash/flatten'; -import keys from 'lodash/keys'; import { resolve as pResolve } from 'node:path'; import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 551ff7822f..940395cdc5 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -1,12 +1,12 @@ -import isEmpty from 'lodash/isEmpty'; -import values from 'lodash/values'; import keys from 'lodash/keys'; -import flatten from 'lodash/flatten'; import { join } from 'node:path'; +import values from 'lodash/values'; +import isEmpty from 'lodash/isEmpty'; +import flatten from 'lodash/flatten'; import { configHandler } from '@contentstack/cli-utilities'; -import { log, formatError, fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; +import { log, formatError, fsUtil, fileHelper } from '../../utils'; import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; //NOTE: Temp types need to remove once sdk available From 334b007ed4f3cf8cf3c950ed981206393d89887a Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 26 Sep 2023 17:26:26 +0530 Subject: [PATCH 4/4] refactor: removed limit from taxonomy and term types --- packages/contentstack-import/src/import/modules/taxonomies.ts | 2 +- packages/contentstack-import/src/types/index.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 940395cdc5..37cade6c80 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -181,7 +181,7 @@ export default class ImportTaxonomies extends BaseClass { serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { const { apiData: taxonomy } = apiOptions; if (this.taxonomiesSuccess.hasOwnProperty(taxonomy.uid)) { - log(this.importConfig, `Taxonomy '${taxonomy.title}' already exists. Skipping it to avoid duplicates!`, 'info'); + log(this.importConfig, `Taxonomy '${taxonomy.name}' already exists. Skipping it to avoid duplicates!`, 'info'); apiOptions.entity = undefined; } else { apiOptions.apiData = taxonomy; diff --git a/packages/contentstack-import/src/types/index.ts b/packages/contentstack-import/src/types/index.ts index 53a6b1f61e..b9d5d5111a 100644 --- a/packages/contentstack-import/src/types/index.ts +++ b/packages/contentstack-import/src/types/index.ts @@ -93,14 +93,12 @@ export interface TaxonomiesConfig{ dirName: string; fileName: string; dependencies?: Modules[]; - limit?: number; } export interface TermsConfig{ dirName: string; fileName: string; dependencies?: Modules[]; - limit?: number; } export { default as DefaultConfig } from './default-config';