Merge pull request #1052 from contentstack/feat/CS-41090-export-taxonomy

Add Taxonomy module to CLI Export Plugin

packages/contentstack-export/src/config/index.ts

@@ -25,6 +25,7 @@ const config: DefaultConfig = {
       'environments',
       'extensions',
       'webhooks',
+      'taxonomies',
       'global-fields',
       'content-types',
       'custom-roles',
@@ -158,6 +159,16 @@ const config: DefaultConfig = {
       dirName: 'marketplace_apps',
       fileName: 'marketplace_apps.json',
     },
+    taxonomies: {
+      dirName: 'taxonomies',
+      fileName: 'taxonomies.json',
+      invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by'],
+    },
+    terms: {
+      dirName: 'terms',
+      fileName: 'terms.json',
+      invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by'],
+    },
   },
   languagesCode: [
     'af-za',

packages/contentstack-export/src/export/modules/taxonomies.ts

@@ -0,0 +1,210 @@
+import omit from 'lodash/omit';
+import isEmpty from 'lodash/isEmpty';
+import flatten from 'lodash/flatten';
+import keys from 'lodash/keys';
+import { resolve as pResolve } from 'node:path';
+import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities';
+
+import BaseClass from './base-class';
+import { log, fsUtil } from '../../utils';
+import { TaxonomiesConfig, TermsConfig, ModuleClassParams } from '../../types';
+
+//NOTE: Temp types need to remove once sdk available
+type TaxonomyPayload = {
+  baseUrl: string;
+  url: string;
+  mgToken: string;
+  apiKey: string;
+};
+
+export default class ExportTaxonomies extends BaseClass {
+  private taxonomies: Record<string, Record<string, string>>;
+  private terms: Record<string, Record<string, string>>;
+  private taxonomiesConfig: TaxonomiesConfig;
+  private taxonomyPayload: TaxonomyPayload;
+  private termsConfig: TermsConfig;
+  public taxonomiesFolderPath: string;
+  public termsFolderPath: string;
+
+  constructor({ exportConfig, stackAPIClient }: ModuleClassParams) {
+    super({ exportConfig, stackAPIClient });
+    this.taxonomies = {};
+    this.terms = {};
+    this.taxonomiesConfig = exportConfig.modules.taxonomies;
+    this.termsConfig = exportConfig.modules.terms;
+    this.taxonomyPayload = {
+      baseUrl: '',
+      url: '',
+      mgToken: exportConfig.management_token,
+      apiKey: exportConfig.source_stack,
+    };
+  }
+
+  async start(): Promise<void> {
+    log(this.exportConfig, 'Starting taxonomies export', 'info');
+
+    //create taxonomies and terms folder in data directory path
+    this.taxonomiesFolderPath = pResolve(
+      this.exportConfig.data,
+      this.exportConfig.branchName || '',
+      this.taxonomiesConfig.dirName,
+    );
+    await fsUtil.makeDirectory(this.taxonomiesFolderPath);
+    this.termsFolderPath = pResolve(this.taxonomiesFolderPath, this.termsConfig.dirName);
+    await fsUtil.makeDirectory(this.termsFolderPath);
+
+    const { cma } = configHandler.get('region') || {};
+    this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`;
+    this.taxonomyPayload.url = this.taxonomyPayload.baseUrl;
+
+    //fetch all taxonomies and write into taxonomies folder
+    await this.getAllTaxonomies(this.taxonomyPayload);
+    if (this.taxonomies === undefined || isEmpty(this.taxonomies)) {
+      log(this.exportConfig, 'No taxonomies found', 'info');
+      return;
+    } else {
+      fsUtil.writeFile(pResolve(this.taxonomiesFolderPath, this.taxonomiesConfig.fileName), this.taxonomies);
+      log(this.exportConfig, 'All the taxonomies have been exported successfully!', 'success');
+    }
+
+    //fetch all terms of respective and write into taxonomies/terms folder
+    await this.getAllTerms();
+  }
+
+  /**
+   * fetch all taxonomies in the provided stack
+   * @param {TaxonomyPayload} payload
+   * @param {number} skip
+   * @returns
+   */
+  async getAllTaxonomies(payload: TaxonomyPayload, skip = 0): Promise<any> {
+    const response = await this.apiRequestHandler(payload, skip);
+    if (response?.taxonomies) {
+      skip += this.taxonomiesConfig.limit || 100;
+      this.sanitizeTaxonomiesAttribs(response.taxonomies);
+      if (skip >= response?.count) {
+        return;
+      } else {
+        return await this.getAllTaxonomies(payload, skip);
+      }
+    }
+    return;
+  }
+
+  /**
+   * remove invalid keys and write data into taxonomies
+   * @function sanitizeTaxonomiesAttribs
+   * @param taxonomies
+   */
+  sanitizeTaxonomiesAttribs(taxonomies: Record<string, string>[]) {
+    for (let index = 0; index < taxonomies?.length; index++) {
+      const taxonomyUID = taxonomies[index].uid;
+      const taxonomyName = taxonomies[index]?.name;
+      this.taxonomies[taxonomyUID] = omit(taxonomies[index], this.taxonomiesConfig.invalidKeys);
+      log(this.exportConfig, `'${taxonomyName}' taxonomy was exported successfully`, 'success');
+    }
+  }
+
+  /**
+   * fetch all terms of respective taxonomy and write it into <taxonomy-uid>-terms file
+   */
+  async getAllTerms() {
+    const taxonomiesUID = keys(this.taxonomies) || [];
+    for (let index = 0; index < taxonomiesUID?.length; index++) {
+      const taxonomyUID = taxonomiesUID[index];
+      this.taxonomyPayload.url = `${this.taxonomyPayload.baseUrl}/${taxonomyUID}/terms`;
+      this.terms = {};
+      await this.fetchTermsOfTaxonomy(this.taxonomyPayload);
+
+      if (this.terms === undefined || isEmpty(this.terms)) {
+        log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'`, 'info');
+      } else {
+        fsUtil.writeFile(pResolve(this.termsFolderPath, `${taxonomyUID}-${this.termsConfig.fileName}`), this.terms);
+        log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' were successfully exported.`, 'success');
+      }
+    }
+    log(this.exportConfig, `All the terms have been exported successfully!`, 'success');
+  }
+
+  /**
+   * fetch all terms of the provided taxonomy uid
+   * @param {TaxonomyPayload} payload
+   * @param {number} skip
+   * @returns
+   */
+  async fetchTermsOfTaxonomy(payload: TaxonomyPayload, skip = 0): Promise<any> {
+    const response = await this.apiRequestHandler(payload, skip);
+    if (response?.terms) {
+      skip += this.termsConfig.limit || 100;
+      this.sanitizeTermsAttribs(response.terms);
+      if (skip >= response?.count) {
+        return;
+      } else {
+        return await this.fetchTermsOfTaxonomy(payload, skip);
+      }
+    }
+    return;
+  }
+
+  /**
+   * remove invalid keys and write data into taxonomies
+   * @function sanitizeTaxonomiesAttribs
+   * @param terms
+   */
+  sanitizeTermsAttribs(terms: Record<string, string>[]) {
+    for (let index = 0; index < terms?.length; index++) {
+      const termUID = terms[index]?.uid;
+      this.terms[termUID] = omit(terms[index], this.termsConfig.invalidKeys);
+    }
+  }
+
+  //NOTE: Temp code need to remove once sdk available
+  async apiRequestHandler(payload: TaxonomyPayload, skip: number) {
+    const headers: any = {
+      api_key: payload.apiKey,
+      'Content-Type': 'application/json',
+    };
+
+    if (payload?.mgToken) headers['authorization'] = payload.mgToken;
+    else headers['authToken'] = configHandler.get('authtoken');
+
+    const params = {
+      include_count: true,
+      skip: 0,
+      limit: this.taxonomiesConfig.limit || 100,
+    };
+
+    if (skip >= 0) params['skip'] = skip;
+
+    return await new HttpClient()
+      .headers(headers)
+      .queryParams(params)
+      .get(payload.url)
+      .then((res: any) => {
+        //NOTE - temporary code for handling api errors response
+        const { status, data } = res;
+        if ([200, 201, 202].includes(status)) return data;
+        else {
+          let errorMsg;
+          if ([500, 503, 502].includes(status)) errorMsg = data?.message || data;
+          else errorMsg = data?.error_message;
+          if (errorMsg === undefined) {
+            errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors));
+          }
+          cliux.print(`Error: ${errorMsg}`, { color: 'red' });
+        }
+      })
+      .catch((err: any) => this.handleErrorMsg(err));
+  }
+
+  //NOTE: Temp code need to remove once sdk available
+  handleErrorMsg(err: any) {
+    if (err?.errorMessage) {
+      cliux.print(`Error: ${err.errorMessage}`, { color: 'red' });
+    } else if (err?.message) {
+      cliux.print(`Error: ${err.message}`, { color: 'red' });
+    } else {
+      cliux.print(`Error: Something went wrong. Please try again.`, { color: 'red' });
+    }
+  }
+}

packages/contentstack-export/src/types/default-config.ts

@@ -143,6 +143,18 @@ export default interface DefaultConfig {
       fileName: string;
       requiredKeys: string[];
     };
+    taxonomies: {
+      dirName: string;
+      fileName: string;
+      invalidKeys: string[];
+      dependencies?: Modules[];
+    };
+    terms: {
+      dirName: string;
+      fileName: string;
+      invalidKeys: string[];
+      dependencies?: Modules[];
+    };
   };
   languagesCode: string[];
   updatedModules: string[];

packages/contentstack-export/src/types/index.ts

@@ -40,7 +40,8 @@ export type Modules =
   | 'custom-roles'
   | 'workflows'
   | 'labels'
-  | 'marketplace-apps';
+  | 'marketplace-apps'
+  | 'taxonomies';
 
 export type ModuleClassParams = {
   stackAPIClient: ReturnType<ContentstackClient['stack']>;
@@ -121,5 +122,21 @@ export interface StackConfig{
     limit?: number;
 }
 
+export interface TaxonomiesConfig{
+  dirName: string;
+  fileName: string;
+  invalidKeys: string[];
+  dependencies?: Modules[];
+  limit?: number;
+}
+
+export interface TermsConfig{
+  dirName: string;
+  fileName: string;
+  invalidKeys: string[];
+  dependencies?: Modules[];
+  limit?: number;
+}
+
 export { default as DefaultConfig } from './default-config';
 export { default as ExportConfig } from './export-config';