diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index 86557f6a8b..049fd0eedb 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -7,7 +7,7 @@ import * as path from 'path'; import { isEmpty, find, cloneDeep, map } from 'lodash'; -import { fsUtil, log, formatError, schemaTemplate, lookupExtension } from '../../utils'; +import { fsUtil, log, formatError, schemaTemplate, lookupExtension, lookUpTaxonomy } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; import { updateFieldRules } from '../../utils/content-type-helper'; @@ -49,6 +49,8 @@ export default class ContentTypesImport extends BaseClass { limit: number; writeConcurrency?: number; }; + private taxonomiesPath: string; + public taxonomies: Record; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -75,6 +77,7 @@ export default class ContentTypesImport extends BaseClass { this.gFs = []; this.createdGFs = []; this.pendingGFs = []; + this.taxonomiesPath = path.join(importConfig.data, 'mapper/taxonomies', 'success.json'); } async start(): Promise { @@ -85,7 +88,6 @@ export default class ContentTypesImport extends BaseClass { * Update pending global fields * write field rules */ - this.cTs = fsUtil.readFile(path.join(this.cTsFolderPath, 'schema.json')) as Record[]; if (!this.cTs || isEmpty(this.cTs)) { log(this.importConfig, 'No content type found to import', 'info'); @@ -95,6 +97,7 @@ export default class ContentTypesImport extends BaseClass { this.installedExtensions = ( ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} } ).extension_uid; + this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record; await this.seedCTs(); log(this.importConfig, 'Created content types', 'success'); @@ -152,7 +155,7 @@ export default class ContentTypesImport extends BaseClass { async updateCTs(): Promise { const onSuccess = ({ response: contentType, apiData: { uid } }: any) => { - log(this.importConfig, `${uid} updated with references`, 'success'); + log(this.importConfig, `'${uid}' updated with references`, 'success'); }; const onReject = ({ error, apiData: { uid } }: any) => { log(this.importConfig, formatError(error), 'error'); @@ -186,6 +189,8 @@ export default class ContentTypesImport extends BaseClass { } this.fieldRules.push(contentType.uid); } + //will remove taxonomy if taxonomy doesn't exists in stack + lookUpTaxonomy(contentType.schema, this.taxonomies); lookupExtension( this.importConfig, contentType.schema, diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index fa83c02d06..84d06fd2e0 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -20,6 +20,7 @@ import { lookupEntries, lookupAssets, fileHelper, + lookUpTerms, } from '../../utils'; import { ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; @@ -53,6 +54,8 @@ export default class EntriesImport extends BaseClass { private entriesUidMapper: Record; private envs: Record; private autoCreatedEntries: Record[]; + private taxonomiesPath: string; + public taxonomies: Record; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -64,6 +67,7 @@ export default class EntriesImport extends BaseClass { this.uniqueUidMapperPath = path.join(this.entriesMapperPath, 'unique-mapping.json'); this.modifiedCTsPath = path.join(this.entriesMapperPath, 'modified-schemas.json'); this.marketplaceAppMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json'); + this.taxonomiesPath = path.join(this.importConfig.data, 'mapper', 'taxonomies', 'terms', 'success.json'); this.entriesConfig = importConfig.modules.entries; this.entriesPath = path.resolve(importConfig.data, this.entriesConfig.dirName); this.cTsPath = path.resolve(importConfig.data, importConfig.modules['content-types'].dirName); @@ -96,6 +100,8 @@ export default class EntriesImport extends BaseClass { this.assetUidMapper = (fsUtil.readFile(this.assetUidMapperPath) as Record) || {}; this.assetUrlMapper = (fsUtil.readFile(this.assetUrlMapperPath) as Record) || {}; + + this.taxonomies = (fsUtil.readFile(this.taxonomiesPath) as Record); fsUtil.makeDirectory(this.entriesMapperPath); await this.disableMandatoryCTReferences(); @@ -360,6 +366,8 @@ export default class EntriesImport extends BaseClass { if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) { entry = removeEntryRefsFromJSONRTE(entry, contentType.schema); } + //will remove term if term doesn't exists in taxonomy + lookUpTerms(contentType?.schema, entry, this.taxonomies); // will replace all old asset uid/urls with new ones entry = lookupAssets( { diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 37cade6c80..fa80f66a34 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -1,4 +1,5 @@ import keys from 'lodash/keys'; +import pick from 'lodash/pick'; import { join } from 'node:path'; import values from 'lodash/values'; import isEmpty from 'lodash/isEmpty'; @@ -13,7 +14,7 @@ import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; type TaxonomyPayload = { baseUrl: string; url: string; - mgToken: string; + mgToken?: string; reqPayload: Record; headers: Record; }; @@ -118,7 +119,7 @@ export default class ImportTaxonomies extends BaseClass { return; } - const apiContent = values(this.taxonomies); + const apiContent = values(this.taxonomies) as Record[];; this.taxonomyUIDs = keys(this.taxonomies); const onSuccess = ({ @@ -128,10 +129,10 @@ export default class ImportTaxonomies extends BaseClass { //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready if ([200, 201, 202].includes(status)) { const { taxonomy } = data; - this.taxonomiesSuccess[taxonomy.uid] = taxonomy; + this.taxonomiesSuccess[taxonomy.uid] = pick(taxonomy, ['name', 'description']); log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); } else { - let errorMsg; + let errorMsg:any; if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; else errorMsg = data?.error_message; if (errorMsg === undefined) { @@ -223,7 +224,7 @@ export default class ImportTaxonomies extends BaseClass { if ([200, 201, 202].includes(status)) { if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; const { term } = data; - this.termsSuccess[taxonomy_uid][term.uid] = term; + this.termsSuccess[taxonomy_uid][term.uid] = pick(term, ['name']); log(this.importConfig, `Term '${name}' imported successfully`, 'success'); } else { if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; @@ -259,7 +260,7 @@ export default class ImportTaxonomies extends BaseClass { ) as Record; if (this.terms !== undefined && !isEmpty(this.terms)) { - const apiContent = values(this.terms); + const apiContent = values(this.terms) as Record[]; await this.makeConcurrentCall( { apiContent, diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index 19cfca611e..c76ee3340d 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -27,3 +27,4 @@ export { restoreJsonRteEntryRefs, } from './entries-helper'; export * from './common-helper'; +export { lookUpTaxonomy, lookUpTerms } from './taxonomies-helper'; diff --git a/packages/contentstack-import/src/utils/taxonomies-helper.ts b/packages/contentstack-import/src/utils/taxonomies-helper.ts new file mode 100644 index 0000000000..ea046c7643 --- /dev/null +++ b/packages/contentstack-import/src/utils/taxonomies-helper.ts @@ -0,0 +1,114 @@ +/** + * taxonomy lookup + */ +import { cliux } from '@contentstack/cli-utilities'; + +/** + * check and remove if referenced taxonomy doesn't exists in stack + * @param {any} schema content type schema + * @param {Record} taxonomies created taxonomies + */ +export const lookUpTaxonomy = function (schema: any, taxonomies: Record) { + for (let i in schema) { + if (schema[i].data_type === 'taxonomy') { + const taxonomyFieldData = schema[i].taxonomies as Record[]; + const { updatedTaxonomyData, isTaxonomyFieldRemoved } = verifyAndRemoveTaxonomy(taxonomyFieldData, taxonomies); + + //Handle API error -> The 'taxonomies' property must have atleast one taxonomy object. Remove taxonomy field from schema. + if (isTaxonomyFieldRemoved) { + schema.splice(i, 1); + } else { + schema[i].taxonomies = updatedTaxonomyData; + } + } + } +}; + +/** + * verify and remove referenced taxonomy with warning from respective content type + * @param {Record[]} taxonomyFieldData + * @param {Record} taxonomies created taxonomies + * @returns + */ +const verifyAndRemoveTaxonomy = function ( + taxonomyFieldData: Record[], + taxonomies: Record, +): { + updatedTaxonomyData: Record[]; + isTaxonomyFieldRemoved: boolean; +} { + let isTaxonomyFieldRemoved: boolean = false; + + for (let index = 0; index < taxonomyFieldData?.length; index++) { + const taxonomyData = taxonomyFieldData[index]; + + if (taxonomies === undefined || !taxonomies.hasOwnProperty(taxonomyData?.taxonomy_uid)) { + // remove taxonomy from taxonomies field data with warning if respective taxonomy doesn't exists + cliux.print(`Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from taxonomies field`, { + color: 'yellow', + }); + taxonomyFieldData.splice(index, 1); + --index; + } + } + + if (!taxonomyFieldData?.length) { + cliux.print('Taxonomy does not exist. Removing the field from content type', { color: 'yellow' }); + isTaxonomyFieldRemoved = true; + } + + return { + updatedTaxonomyData: taxonomyFieldData, + isTaxonomyFieldRemoved, + }; +}; + +/** + * check and remove if referenced terms doesn't exists in taxonomy + * @param {Record[]} ctSchema content type schema + * @param {any} entry + * @param {Record} taxonomiesAndTermData created taxonomies and terms + */ +export const lookUpTerms = function ( + ctSchema: Record[], + entry: any, + taxonomiesAndTermData: Record, +) { + for (let index = 0; index < ctSchema?.length; index++) { + if (ctSchema[index].data_type === 'taxonomy') { + const taxonomyFieldData = entry[ctSchema[index].uid]; + const updatedTaxonomyData = verifyAndRemoveTerms(taxonomyFieldData, taxonomiesAndTermData); + entry[ctSchema[index].uid] = updatedTaxonomyData; + } + } +}; + +/** + * verify and remove referenced term with warning from respective entry + * @param {Record[]} taxonomyFieldData entry taxonomies data + * @param {Record} taxonomiesAndTermData created taxonomies and terms + * @returns { Record[]} + */ +const verifyAndRemoveTerms = function ( + taxonomyFieldData: Record[], + taxonomiesAndTermData: Record, +): Record[] { + for (let index = 0; index < taxonomyFieldData?.length; index++) { + const taxonomyData = taxonomyFieldData[index]; + const taxUID = taxonomyData?.taxonomy_uid; + const termUID = taxonomyData?.term_uid; + + if ( + taxonomiesAndTermData === undefined || + !taxonomiesAndTermData.hasOwnProperty(taxUID) || + (taxonomiesAndTermData.hasOwnProperty(taxUID) && !taxonomiesAndTermData[taxUID].hasOwnProperty(termUID)) + ) { + // remove term from taxonomies field data with warning if respective term doesn't exists + cliux.print(`Term '${termUID}' does not exist. Removing it from taxonomy - '${taxUID}'`, { color: 'yellow' }); + taxonomyFieldData.splice(index, 1); + --index; + } + } + + return taxonomyFieldData; +};