diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index c9003f3622..3a53ae0baf 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -28,6 +28,7 @@ const config: DefaultConfig = { 'assets', 'extensions', 'marketplace-apps', + 'taxonomies', 'global-fields', 'content-types', 'custom-roles', @@ -142,6 +143,14 @@ const config: DefaultConfig = { dirName: 'marketplace_apps', fileName: 'marketplace_apps.json', }, + taxonomies: { + dirName: 'taxonomies', + fileName: 'taxonomies.json' + }, + terms: { + dirName: 'terms', + fileName: 'terms.json' + }, }, languagesCode: [ 'af-za', diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index fa99462ead..e47f8790ab 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -18,6 +18,7 @@ import { LabelData } from '@contentstack/management/types/stack/label'; import { WebhookData } from '@contentstack/management/types/stack/webhook'; import { WorkflowData } from '@contentstack/management/types/stack/workflow'; import { RoleData } from '@contentstack/management/types/stack/role'; +import { HttpClient } from '@contentstack/cli-utilities'; import { log } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; @@ -47,7 +48,9 @@ export type ApiModuleType = | 'create-entries' | 'update-entries' | 'publish-entries' - | 'delete-entries'; + | 'delete-entries' + | 'create-taxonomies' + | 'create-terms'; export type ApiOptions = { uid?: string; @@ -381,6 +384,19 @@ export default abstract class BaseClass { .delete({ locale: additionalInfo.locale }) .then(onSuccess) .catch(onReject); + case 'create-taxonomies': + return new HttpClient() + .headers(additionalInfo.headers) + .post(additionalInfo.url, { taxonomy: apiData }) + .then(onSuccess) + .catch(onReject); + case 'create-terms': + const url = `${additionalInfo.baseUrl}/${apiData.taxonomy_uid}/terms`; + return new HttpClient() + .headers(additionalInfo.headers) + .post(url, { term: apiData }) + .then(onSuccess) + .catch(onReject); default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts new file mode 100644 index 0000000000..d0b5b9e820 --- /dev/null +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -0,0 +1,247 @@ +import isEmpty from 'lodash/isEmpty'; +import values from 'lodash/values'; +import keys from 'lodash/keys'; +import flatten from 'lodash/flatten'; +import { join, resolve as pResolve } from 'node:path'; +import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; + +import config from '../../config'; +import { log, formatError, fsUtil, fileHelper } from '../../utils'; +import BaseClass, { ApiOptions } from './base-class'; +import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; + +//NOTE: Temp types need to remove once sdk available +type TaxonomyPayload = { + baseUrl: string; + url: string; + mgToken: string; + reqPayload: Record; + headers: Record; +}; + +export default class ImportTaxonomies extends BaseClass { + private taxonomiesMapperDirPath: string; + private termsMapperDirPath: string; + private taxonomiesFolderPath: string; + private termsFolderPath: string; + private taxSuccessPath: string; + private taxFailsPath: string; + private taxonomiesConfig: TaxonomiesConfig; + private termsConfig: TermsConfig; + private taxonomies: Record; + private taxonomiesSuccess: Record = {}; + private taxonomiesFailed: Record = {}; + private taxonomyPayload: TaxonomyPayload; + + constructor({ importConfig, stackAPIClient }: ModuleClassParams) { + super({ importConfig, stackAPIClient }); + this.taxonomiesConfig = importConfig.modules.taxonomies; + this.termsConfig = importConfig.modules.terms; + this.taxonomiesMapperDirPath = join(importConfig.backupDir, 'mapper', 'taxonomies'); + this.termsMapperDirPath = join(this.taxonomiesMapperDirPath, 'terms'); + this.taxonomiesFolderPath = join(importConfig.backupDir, this.taxonomiesConfig.dirName); + this.termsFolderPath = join(this.taxonomiesFolderPath, this.termsConfig.dirName); + this.taxSuccessPath = join(this.taxonomiesMapperDirPath, 'success.json'); + this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json'); + this.taxonomyPayload = { + baseUrl: '', + url: '', + mgToken: importConfig.management_token, + reqPayload: {}, + headers: { + 'Content-Type': 'application/json', + api_key: importConfig.target_stack, + }, + }; + } + + /** + * @method start + * @returns {Promise} Promise + */ + async start(): Promise { + log(this.importConfig, 'Migrating taxonomies', 'info'); + + //Step1 check folder exists or not + if (fileHelper.fileExistsSync(this.taxonomiesFolderPath)) { + this.taxonomies = fsUtil.readFile(join(this.taxonomiesFolderPath, 'taxonomies.json'), true) as Record< + string, + unknown + >; + } else { + log(this.importConfig, `No such file or directory - '${this.taxonomiesFolderPath}'`, 'error'); + return; + } + + //NOTE - Temp code for api request + const { cma } = configHandler.get('region') || {}; + this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; + this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; + if (this.taxonomyPayload?.mgToken) this.taxonomyPayload.headers['authorization'] = this.taxonomyPayload.mgToken; + else this.taxonomyPayload.headers['authtoken'] = configHandler.get('authtoken'); + + //Step 2 create taxonomies & terms mapper directory + await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); + await fsUtil.makeDirectory(this.termsMapperDirPath); + + await this.importTaxonomies(); + // create terms related to respective taxonomy + if (!fileHelper.fileExistsSync(this.termsFolderPath)) { + log(this.importConfig, `No such file or directory - '${this.taxonomiesFolderPath}'`, 'error'); + return; + } + await this.importTerms(); + + if (this.taxonomiesSuccess !== undefined && !isEmpty(this.taxonomiesSuccess)) { + fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesSuccess); + } + + if (this.taxonomiesFailed !== undefined &&!isEmpty(this.taxonomiesFailed)) { + fsUtil.writeFile(this.taxFailsPath, this.taxonomiesFailed); + } + + log(this.importConfig, 'Taxonomies have been imported successfully!', 'success'); + } + + async importTaxonomies(): Promise { + if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { + log(this.importConfig, 'No Taxonomies Found', 'info'); + return; + } + + const apiContent = values(this.taxonomies); + + const onSuccess = ({ + response: { data, status } = { data: null, status: null }, + apiData: { uid, name } = { uid: null, name: '' }, + }: any) => { + //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready + if ([200, 201, 202].includes(status)) { + this.taxonomiesSuccess[uid] = data; + log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); + } else { + let errorMsg; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`; + log(this.importConfig, `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error'); + } + }; + + const onReject = ({ error, apiData }: any) => { + const err = error?.message ? JSON.parse(error.message) : error; + const { name } = apiData; + if (err?.errors?.name) { + log(this.importConfig, `Taxonomy '${name}' already exists`, 'info'); + } else { + this.taxonomiesFailed[apiData.uid] = apiData; + log(this.importConfig, `Taxonomy '${name}' failed to be import ${formatError(error)}`, 'error'); + log(this.importConfig, error, 'error'); + } + }; + + await this.makeConcurrentCall( + { + apiContent, + processName: 'import taxonomies', + apiParams: { + serializeData: this.serializeTaxonomy.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'create-taxonomies', + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, + }, + concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, + }, + undefined, + false, + ); + } + + /** + * @method serializeTaxonomy + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { + const { apiData: taxonomy } = apiOptions; + if (this.taxonomiesSuccess.hasOwnProperty(taxonomy.uid)) { + log(this.importConfig, `Taxonomy '${taxonomy.title}' already exists. Skipping it to avoid duplicates!`, 'info'); + apiOptions.entity = undefined; + } else { + apiOptions.apiData = taxonomy; + } + apiOptions.apiData = taxonomy; + return apiOptions; + } + + async importTerms(): Promise { + if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { + return; + } + + const listOfTaxonomyUIDs = keys(this.taxonomies); + + const onSuccess = ({ + response, + apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: '' }, + }: any) => { + this.taxonomiesSuccess[uid] = response; + log(this.importConfig, `Term '${name}' imported successfully`, 'success'); + }; + + const onReject = ({ error, apiData }: any) => { + const err = error?.message ? JSON.parse(error.message) : error; + const { name } = apiData; + if (err?.errors?.name) { + log(this.importConfig, `Term '${name}' already exists`, 'info'); + } else { + this.taxonomiesSuccess[apiData.uid] = apiData; + log(this.importConfig, `Term '${name}' failed to be import ${formatError(error)}`, 'error'); + log(this.importConfig, error, 'error'); + } + }; + + for (const taxUID of listOfTaxonomyUIDs) { + const terms = fsUtil.readFile( + join(this.termsFolderPath, `${taxUID}-${this.termsConfig.fileName}`), + true, + ) as Record; + const dirPath = pResolve(this.termsMapperDirPath, `${taxUID}-terms`); + if (!fileHelper.fileExistsSync(dirPath)) { + await fsUtil.makeDirectory(dirPath); + } + //success and fail path for particular taxonomy uid + const apiContent = values(terms); + await this.makeConcurrentCall( + { + apiContent, + processName: 'import terms', + apiParams: { + serializeData: this.serializeTerms.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'create-terms', + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, + }, + concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, + }, + undefined, + false, + ); + } + } + + /** + * @method serializeTerms + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeTerms(apiOptions: ApiOptions): ApiOptions { + const { apiData: term } = apiOptions; + apiOptions.apiData = term; + return apiOptions; + } +} diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index c41b949fdb..27b0af0859 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -113,6 +113,16 @@ export default interface DefaultConfig { fileName: string; requiredKeys: string[]; }; + taxonomies: { + dirName: string; + fileName: string; + dependencies?: Modules[]; + }; + terms: { + dirName: string; + fileName: string; + dependencies?: Modules[]; + }; }; languagesCode: string[]; apis: { diff --git a/packages/contentstack-import/src/types/index.ts b/packages/contentstack-import/src/types/index.ts index 104328215f..53a6b1f61e 100644 --- a/packages/contentstack-import/src/types/index.ts +++ b/packages/contentstack-import/src/types/index.ts @@ -40,7 +40,8 @@ export type Modules = | 'custom-roles' | 'workflows' | 'labels' - | 'marketplace-apps'; + | 'marketplace-apps' + | 'taxonomies'; export type ModuleClassParams = { stackAPIClient: ReturnType; @@ -88,5 +89,19 @@ export interface CustomRoleConfig{ customRolesLocalesFileName: string; } +export interface TaxonomiesConfig{ + dirName: string; + fileName: string; + dependencies?: Modules[]; + limit?: number; +} + +export interface TermsConfig{ + dirName: string; + fileName: string; + dependencies?: Modules[]; + limit?: number; +} + export { default as DefaultConfig } from './default-config'; export { default as ImportConfig } from './import-config';