From f0f880a32163d1ab85e550f011c4ec5fce274a08 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 4 Oct 2023 16:01:43 +0530 Subject: [PATCH] feat: entries module supports taxonomy --- .../src/import/modules/content-types.ts | 4 +- .../src/import/modules/entries.ts | 8 ++ .../src/import/modules/taxonomies.ts | 5 +- .../contentstack-import/src/utils/index.ts | 2 +- .../src/utils/taxonomies-helper.ts | 120 +++++++++++++++--- 5 files changed, 114 insertions(+), 25 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index 0faff4e7ff..4440813098 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -50,7 +50,7 @@ export default class ContentTypesImport extends BaseClass { writeConcurrency?: number; }; private taxonomiesPath: string; - public taxonomies: Record[] = []; + public taxonomies: Record; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -97,7 +97,7 @@ export default class ContentTypesImport extends BaseClass { this.installedExtensions = ( ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} } ).extension_uid; - this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record[]; + this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record; await this.seedCTs(); log(this.importConfig, 'Created content types', 'success'); diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index fa83c02d06..84d06fd2e0 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -20,6 +20,7 @@ import { lookupEntries, lookupAssets, fileHelper, + lookUpTerms, } from '../../utils'; import { ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; @@ -53,6 +54,8 @@ export default class EntriesImport extends BaseClass { private entriesUidMapper: Record; private envs: Record; private autoCreatedEntries: Record[]; + private taxonomiesPath: string; + public taxonomies: Record; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -64,6 +67,7 @@ export default class EntriesImport extends BaseClass { this.uniqueUidMapperPath = path.join(this.entriesMapperPath, 'unique-mapping.json'); this.modifiedCTsPath = path.join(this.entriesMapperPath, 'modified-schemas.json'); this.marketplaceAppMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json'); + this.taxonomiesPath = path.join(this.importConfig.data, 'mapper', 'taxonomies', 'terms', 'success.json'); this.entriesConfig = importConfig.modules.entries; this.entriesPath = path.resolve(importConfig.data, this.entriesConfig.dirName); this.cTsPath = path.resolve(importConfig.data, importConfig.modules['content-types'].dirName); @@ -96,6 +100,8 @@ export default class EntriesImport extends BaseClass { this.assetUidMapper = (fsUtil.readFile(this.assetUidMapperPath) as Record) || {}; this.assetUrlMapper = (fsUtil.readFile(this.assetUrlMapperPath) as Record) || {}; + + this.taxonomies = (fsUtil.readFile(this.taxonomiesPath) as Record); fsUtil.makeDirectory(this.entriesMapperPath); await this.disableMandatoryCTReferences(); @@ -360,6 +366,8 @@ export default class EntriesImport extends BaseClass { if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) { entry = removeEntryRefsFromJSONRTE(entry, contentType.schema); } + //will remove term if term doesn't exists in taxonomy + lookUpTerms(contentType?.schema, entry, this.taxonomies); // will replace all old asset uid/urls with new ones entry = lookupAssets( { diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 80e923e33b..fa80f66a34 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -1,4 +1,5 @@ import keys from 'lodash/keys'; +import pick from 'lodash/pick'; import { join } from 'node:path'; import values from 'lodash/values'; import isEmpty from 'lodash/isEmpty'; @@ -128,7 +129,7 @@ export default class ImportTaxonomies extends BaseClass { //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready if ([200, 201, 202].includes(status)) { const { taxonomy } = data; - this.taxonomiesSuccess[taxonomy.uid] = taxonomy; + this.taxonomiesSuccess[taxonomy.uid] = pick(taxonomy, ['name', 'description']); log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); } else { let errorMsg:any; @@ -223,7 +224,7 @@ export default class ImportTaxonomies extends BaseClass { if ([200, 201, 202].includes(status)) { if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; const { term } = data; - this.termsSuccess[taxonomy_uid][term.uid] = term; + this.termsSuccess[taxonomy_uid][term.uid] = pick(term, ['name']); log(this.importConfig, `Term '${name}' imported successfully`, 'success'); } else { if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index 1cfe9a620e..c76ee3340d 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -27,4 +27,4 @@ export { restoreJsonRteEntryRefs, } from './entries-helper'; export * from './common-helper'; -export { lookUpTaxonomy } from './taxonomies-helper'; +export { lookUpTaxonomy, lookUpTerms } from './taxonomies-helper'; diff --git a/packages/contentstack-import/src/utils/taxonomies-helper.ts b/packages/contentstack-import/src/utils/taxonomies-helper.ts index df9ad6fc48..ea046c7643 100644 --- a/packages/contentstack-import/src/utils/taxonomies-helper.ts +++ b/packages/contentstack-import/src/utils/taxonomies-helper.ts @@ -3,32 +3,112 @@ */ import { cliux } from '@contentstack/cli-utilities'; -export const lookUpTaxonomy = function (schema: any, taxonomies: Record[]) { +/** + * check and remove if referenced taxonomy doesn't exists in stack + * @param {any} schema content type schema + * @param {Record} taxonomies created taxonomies + */ +export const lookUpTaxonomy = function (schema: any, taxonomies: Record) { for (let i in schema) { if (schema[i].data_type === 'taxonomy') { const taxonomyFieldData = schema[i].taxonomies as Record[]; - if (!taxonomyFieldData.length) break; - for (let index = 0; index < taxonomyFieldData.length; index++) { - const taxonomyData = taxonomyFieldData[index]; - if (taxonomies === undefined || !taxonomies[taxonomyData?.taxonomy_uid]) { - // remove taxonomy from taxonomies field data with warning - cliux.print( - `Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from taxonomy field`, - { color: 'yellow' }, - ); - taxonomyFieldData.splice(index, 1); - --index; - } - } - //Case 1:- if no taxonomy exist and trying to create taxonomies field API error -> The 'taxonomies' property must have atleast one taxonomy object. - if (!taxonomyFieldData?.length) { - cliux.print(`Content type related Taxonomy does not exist in stack. Removing the field from schema`, { - color: 'yellow', - }); + const { updatedTaxonomyData, isTaxonomyFieldRemoved } = verifyAndRemoveTaxonomy(taxonomyFieldData, taxonomies); + + //Handle API error -> The 'taxonomies' property must have atleast one taxonomy object. Remove taxonomy field from schema. + if (isTaxonomyFieldRemoved) { schema.splice(i, 1); } else { - schema[i].taxonomies = taxonomyFieldData; + schema[i].taxonomies = updatedTaxonomyData; } } } }; + +/** + * verify and remove referenced taxonomy with warning from respective content type + * @param {Record[]} taxonomyFieldData + * @param {Record} taxonomies created taxonomies + * @returns + */ +const verifyAndRemoveTaxonomy = function ( + taxonomyFieldData: Record[], + taxonomies: Record, +): { + updatedTaxonomyData: Record[]; + isTaxonomyFieldRemoved: boolean; +} { + let isTaxonomyFieldRemoved: boolean = false; + + for (let index = 0; index < taxonomyFieldData?.length; index++) { + const taxonomyData = taxonomyFieldData[index]; + + if (taxonomies === undefined || !taxonomies.hasOwnProperty(taxonomyData?.taxonomy_uid)) { + // remove taxonomy from taxonomies field data with warning if respective taxonomy doesn't exists + cliux.print(`Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from taxonomies field`, { + color: 'yellow', + }); + taxonomyFieldData.splice(index, 1); + --index; + } + } + + if (!taxonomyFieldData?.length) { + cliux.print('Taxonomy does not exist. Removing the field from content type', { color: 'yellow' }); + isTaxonomyFieldRemoved = true; + } + + return { + updatedTaxonomyData: taxonomyFieldData, + isTaxonomyFieldRemoved, + }; +}; + +/** + * check and remove if referenced terms doesn't exists in taxonomy + * @param {Record[]} ctSchema content type schema + * @param {any} entry + * @param {Record} taxonomiesAndTermData created taxonomies and terms + */ +export const lookUpTerms = function ( + ctSchema: Record[], + entry: any, + taxonomiesAndTermData: Record, +) { + for (let index = 0; index < ctSchema?.length; index++) { + if (ctSchema[index].data_type === 'taxonomy') { + const taxonomyFieldData = entry[ctSchema[index].uid]; + const updatedTaxonomyData = verifyAndRemoveTerms(taxonomyFieldData, taxonomiesAndTermData); + entry[ctSchema[index].uid] = updatedTaxonomyData; + } + } +}; + +/** + * verify and remove referenced term with warning from respective entry + * @param {Record[]} taxonomyFieldData entry taxonomies data + * @param {Record} taxonomiesAndTermData created taxonomies and terms + * @returns { Record[]} + */ +const verifyAndRemoveTerms = function ( + taxonomyFieldData: Record[], + taxonomiesAndTermData: Record, +): Record[] { + for (let index = 0; index < taxonomyFieldData?.length; index++) { + const taxonomyData = taxonomyFieldData[index]; + const taxUID = taxonomyData?.taxonomy_uid; + const termUID = taxonomyData?.term_uid; + + if ( + taxonomiesAndTermData === undefined || + !taxonomiesAndTermData.hasOwnProperty(taxUID) || + (taxonomiesAndTermData.hasOwnProperty(taxUID) && !taxonomiesAndTermData[taxUID].hasOwnProperty(termUID)) + ) { + // remove term from taxonomies field data with warning if respective term doesn't exists + cliux.print(`Term '${termUID}' does not exist. Removing it from taxonomy - '${taxUID}'`, { color: 'yellow' }); + taxonomyFieldData.splice(index, 1); + --index; + } + } + + return taxonomyFieldData; +};