Import content types and entries module supports taxonomy

packages/contentstack-import/src/import/modules/content-types.ts

@@ -7,7 +7,7 @@
 
 import * as path from 'path';
 import { isEmpty, find, cloneDeep, map } from 'lodash';
-import { fsUtil, log, formatError, schemaTemplate, lookupExtension } from '../../utils';
+import { fsUtil, log, formatError, schemaTemplate, lookupExtension, lookUpTaxonomy } from '../../utils';
 import { ImportConfig, ModuleClassParams } from '../../types';
 import BaseClass, { ApiOptions } from './base-class';
 import { updateFieldRules } from '../../utils/content-type-helper';
@@ -49,6 +49,8 @@ export default class ContentTypesImport extends BaseClass {
     limit: number;
     writeConcurrency?: number;
   };
+  private taxonomiesPath: string;
+  public taxonomies: Record<string, unknown>;
 
   constructor({ importConfig, stackAPIClient }: ModuleClassParams) {
     super({ importConfig, stackAPIClient });
@@ -75,6 +77,7 @@ export default class ContentTypesImport extends BaseClass {
     this.gFs = [];
     this.createdGFs = [];
     this.pendingGFs = [];
+    this.taxonomiesPath = path.join(importConfig.data, 'mapper/taxonomies', 'success.json');
   }
 
   async start(): Promise<any> {
@@ -85,7 +88,6 @@ export default class ContentTypesImport extends BaseClass {
      * Update pending global fields
      * write field rules
      */
-
     this.cTs = fsUtil.readFile(path.join(this.cTsFolderPath, 'schema.json')) as Record<string, unknown>[];
     if (!this.cTs || isEmpty(this.cTs)) {
       log(this.importConfig, 'No content type found to import', 'info');
@@ -95,6 +97,7 @@ export default class ContentTypesImport extends BaseClass {
     this.installedExtensions = (
       ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} }
     ).extension_uid;
+    this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record<string, unknown>;
 
     await this.seedCTs();
     log(this.importConfig, 'Created content types', 'success');
@@ -152,7 +155,7 @@ export default class ContentTypesImport extends BaseClass {
 
   async updateCTs(): Promise<any> {
     const onSuccess = ({ response: contentType, apiData: { uid } }: any) => {
-      log(this.importConfig, `${uid} updated with references`, 'success');
+      log(this.importConfig, `'${uid}' updated with references`, 'success');
     };
     const onReject = ({ error, apiData: { uid } }: any) => {
       log(this.importConfig, formatError(error), 'error');
@@ -186,6 +189,8 @@ export default class ContentTypesImport extends BaseClass {
       }
       this.fieldRules.push(contentType.uid);
     }
+    //will remove taxonomy if taxonomy doesn't exists in stack
+    lookUpTaxonomy(contentType.schema, this.taxonomies);
     lookupExtension(
       this.importConfig,
       contentType.schema,

packages/contentstack-import/src/import/modules/entries.ts

@@ -20,6 +20,7 @@ import {
   lookupEntries,
   lookupAssets,
   fileHelper,
+  lookUpTerms,
 } from '../../utils';
 import { ModuleClassParams } from '../../types';
 import BaseClass, { ApiOptions } from './base-class';
@@ -53,6 +54,8 @@ export default class EntriesImport extends BaseClass {
   private entriesUidMapper: Record<string, any>;
   private envs: Record<string, any>;
   private autoCreatedEntries: Record<string, any>[];
+  private taxonomiesPath: string;
+  public taxonomies: Record<string, unknown>;
 
   constructor({ importConfig, stackAPIClient }: ModuleClassParams) {
     super({ importConfig, stackAPIClient });
@@ -64,6 +67,7 @@ export default class EntriesImport extends BaseClass {
     this.uniqueUidMapperPath = path.join(this.entriesMapperPath, 'unique-mapping.json');
     this.modifiedCTsPath = path.join(this.entriesMapperPath, 'modified-schemas.json');
     this.marketplaceAppMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json');
+    this.taxonomiesPath = path.join(this.importConfig.data, 'mapper', 'taxonomies', 'terms', 'success.json');
     this.entriesConfig = importConfig.modules.entries;
     this.entriesPath = path.resolve(importConfig.data, this.entriesConfig.dirName);
     this.cTsPath = path.resolve(importConfig.data, importConfig.modules['content-types'].dirName);
@@ -96,6 +100,8 @@ export default class EntriesImport extends BaseClass {
 
       this.assetUidMapper = (fsUtil.readFile(this.assetUidMapperPath) as Record<string, any>) || {};
       this.assetUrlMapper = (fsUtil.readFile(this.assetUrlMapperPath) as Record<string, any>) || {};
+
+      this.taxonomies = (fsUtil.readFile(this.taxonomiesPath) as Record<string, any>);
 
       fsUtil.makeDirectory(this.entriesMapperPath);
       await this.disableMandatoryCTReferences();
@@ -360,6 +366,8 @@ export default class EntriesImport extends BaseClass {
       if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) {
         entry = removeEntryRefsFromJSONRTE(entry, contentType.schema);
       }
+      //will remove term if term doesn't exists in taxonomy
+      lookUpTerms(contentType?.schema, entry, this.taxonomies);
       // will replace all old asset uid/urls with new ones
       entry = lookupAssets(
         {

packages/contentstack-import/src/import/modules/taxonomies.ts

@@ -1,4 +1,5 @@
 import keys from 'lodash/keys';
+import pick from 'lodash/pick';
 import { join } from 'node:path';
 import values from 'lodash/values';
 import isEmpty from 'lodash/isEmpty';
@@ -13,7 +14,7 @@ import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types';
 type TaxonomyPayload = {
   baseUrl: string;
   url: string;
-  mgToken: string;
+  mgToken?: string;
   reqPayload: Record<string, unknown>;
   headers: Record<string, unknown>;
 };
@@ -118,7 +119,7 @@ export default class ImportTaxonomies extends BaseClass {
       return;
     }
 
-    const apiContent = values(this.taxonomies);
+    const apiContent = values(this.taxonomies) as Record<string, any>[];;
     this.taxonomyUIDs = keys(this.taxonomies);
 
     const onSuccess = ({
@@ -128,10 +129,10 @@ export default class ImportTaxonomies extends BaseClass {
       //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready
       if ([200, 201, 202].includes(status)) {
         const { taxonomy } = data;
-        this.taxonomiesSuccess[taxonomy.uid] = taxonomy;
+        this.taxonomiesSuccess[taxonomy.uid] = pick(taxonomy, ['name', 'description']);
         log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success');
       } else {
-        let errorMsg;
+        let errorMsg:any;
         if ([500, 503, 502].includes(status)) errorMsg = data?.message || data;
         else errorMsg = data?.error_message;
         if (errorMsg === undefined) {
@@ -223,7 +224,7 @@ export default class ImportTaxonomies extends BaseClass {
       if ([200, 201, 202].includes(status)) {
         if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {};
         const { term } = data;
-        this.termsSuccess[taxonomy_uid][term.uid] = term;
+        this.termsSuccess[taxonomy_uid][term.uid] = pick(term, ['name']);
         log(this.importConfig, `Term '${name}' imported successfully`, 'success');
       } else {
         if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {};
@@ -259,7 +260,7 @@ export default class ImportTaxonomies extends BaseClass {
       ) as Record<string, unknown>;
 
       if (this.terms !== undefined && !isEmpty(this.terms)) {
-        const apiContent = values(this.terms);
+        const apiContent = values(this.terms) as Record<string, any>[];
         await this.makeConcurrentCall(
           {
             apiContent,

packages/contentstack-import/src/utils/index.ts

@@ -27,3 +27,4 @@ export {
   restoreJsonRteEntryRefs,
 } from './entries-helper';
 export * from './common-helper';
+export { lookUpTaxonomy, lookUpTerms } from './taxonomies-helper';

packages/contentstack-import/src/utils/taxonomies-helper.ts

@@ -0,0 +1,114 @@
+/**
+ * taxonomy lookup
+ */
+import { cliux } from '@contentstack/cli-utilities';
+
+/**
+ * check and remove if referenced taxonomy doesn't exists in stack
+ * @param {any} schema content type schema
+ * @param {Record<string, unknown>} taxonomies created taxonomies
+ */
+export const lookUpTaxonomy = function (schema: any, taxonomies: Record<string, unknown>) {
+  for (let i in schema) {
+    if (schema[i].data_type === 'taxonomy') {
+      const taxonomyFieldData = schema[i].taxonomies as Record<string, any>[];
+      const { updatedTaxonomyData, isTaxonomyFieldRemoved } = verifyAndRemoveTaxonomy(taxonomyFieldData, taxonomies);
+
+      //Handle API error -> The 'taxonomies' property must have atleast one taxonomy object. Remove taxonomy field from schema.
+      if (isTaxonomyFieldRemoved) {
+        schema.splice(i, 1);
+      } else {
+        schema[i].taxonomies = updatedTaxonomyData;
+      }
+    }
+  }
+};
+
+/**
+ * verify and remove referenced taxonomy with warning from respective content type
+ * @param {Record<string, any>[]} taxonomyFieldData
+ * @param {Record<string, unknown>} taxonomies created taxonomies
+ * @returns
+ */
+const verifyAndRemoveTaxonomy = function (
+  taxonomyFieldData: Record<string, any>[],
+  taxonomies: Record<string, unknown>,
+): {
+  updatedTaxonomyData: Record<string, any>[];
+  isTaxonomyFieldRemoved: boolean;
+} {
+  let isTaxonomyFieldRemoved: boolean = false;
+
+  for (let index = 0; index < taxonomyFieldData?.length; index++) {
+    const taxonomyData = taxonomyFieldData[index];
+
+    if (taxonomies === undefined || !taxonomies.hasOwnProperty(taxonomyData?.taxonomy_uid)) {
+      // remove taxonomy from taxonomies field data with warning if respective taxonomy doesn't exists
+      cliux.print(`Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from taxonomies field`, {
+        color: 'yellow',
+      });
+      taxonomyFieldData.splice(index, 1);
+      --index;
+    }
+  }
+
+  if (!taxonomyFieldData?.length) {
+    cliux.print('Taxonomy does not exist. Removing the field from content type', { color: 'yellow' });
+    isTaxonomyFieldRemoved = true;
+  }
+
+  return {
+    updatedTaxonomyData: taxonomyFieldData,
+    isTaxonomyFieldRemoved,
+  };
+};
+
+/**
+ * check and remove if referenced terms doesn't exists in taxonomy
+ * @param {Record<string, any>[]} ctSchema content type schema
+ * @param {any} entry
+ * @param {Record<string, any>} taxonomiesAndTermData created taxonomies and terms
+ */
+export const lookUpTerms = function (
+  ctSchema: Record<string, any>[],
+  entry: any,
+  taxonomiesAndTermData: Record<string, any>,
+) {
+  for (let index = 0; index < ctSchema?.length; index++) {
+    if (ctSchema[index].data_type === 'taxonomy') {
+      const taxonomyFieldData = entry[ctSchema[index].uid];
+      const updatedTaxonomyData = verifyAndRemoveTerms(taxonomyFieldData, taxonomiesAndTermData);
+      entry[ctSchema[index].uid] = updatedTaxonomyData;
+    }
+  }
+};
+
+/**
+ * verify and remove referenced term with warning from respective entry
+ * @param {Record<string, any>[]} taxonomyFieldData entry taxonomies data
+ * @param {Record<string, any>} taxonomiesAndTermData created taxonomies and terms
+ * @returns { Record<string, any>[]}
+ */
+const verifyAndRemoveTerms = function (
+  taxonomyFieldData: Record<string, any>[],
+  taxonomiesAndTermData: Record<string, any>,
+): Record<string, any>[] {
+  for (let index = 0; index < taxonomyFieldData?.length; index++) {
+    const taxonomyData = taxonomyFieldData[index];
+    const taxUID = taxonomyData?.taxonomy_uid;
+    const termUID = taxonomyData?.term_uid;
+
+    if (
+      taxonomiesAndTermData === undefined ||
+      !taxonomiesAndTermData.hasOwnProperty(taxUID) ||
+      (taxonomiesAndTermData.hasOwnProperty(taxUID) && !taxonomiesAndTermData[taxUID].hasOwnProperty(termUID))
+    ) {
+      // remove term from taxonomies field data with warning if respective term doesn't exists
+      cliux.print(`Term '${termUID}' does not exist. Removing it from taxonomy - '${taxUID}'`, { color: 'yellow' });
+      taxonomyFieldData.splice(index, 1);
+      --index;
+    }
+  }
+
+  return taxonomyFieldData;
+};