Skip to content

Commit

Permalink
feat: import terms
Browse files Browse the repository at this point in the history
  • Loading branch information
@aman19K
aman19K committed Sep 26, 2023
1 parent c5da4ab commit 3344ff9
Show file tree
Hide file tree
Showing 5 changed files with 119 additions and 57 deletions.
2 changes: 1 addition & 1 deletion packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ class ExportToCsvCommand extends Command {
throw new Error(branchExists.errorMessage);
}
stack.branch_uid = branchUid;
stackAPIClient = getStackClient(managementAPIClient, stack);
stackAPIClient = this.getStackClient(managementAPIClient, stack);
} catch (error) {
if (error?.message || error?.errorMessage) {
cliux.error(util.formatError(error));
Expand Down
3 changes: 2 additions & 1 deletion packages/contentstack-export/src/export/modules/taxonomies.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ export default class ExportTaxonomies extends BaseClass {
log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'`, 'info');
} else {
fsUtil.writeFile(pResolve(this.termsFolderPath, `${taxonomyUID}-${this.termsConfig.fileName}`), this.terms);
log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' were successfully exported.`, 'success');
log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' were exported successfully.`, 'success');
}
}
log(this.exportConfig, `All the terms have been exported successfully!`, 'success');
Expand Down Expand Up @@ -172,6 +172,7 @@ export default class ExportTaxonomies extends BaseClass {
include_count: true,
skip: 0,
limit: this.taxonomiesConfig.limit || 100,
depth: 0 // include all the terms if set to 0
};

if (skip >= 0) params['skip'] = skip;
Expand Down
2 changes: 1 addition & 1 deletion packages/contentstack-import/src/config/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ const config: DefaultConfig = {
'locales',
'environments',
'assets',
'taxonomies',
'extensions',
'marketplace-apps',
'taxonomies',
'global-fields',
'content-types',
'custom-roles',
Expand Down
167 changes: 114 additions & 53 deletions packages/contentstack-import/src/import/modules/taxonomies.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@ import isEmpty from 'lodash/isEmpty';
import values from 'lodash/values';
import keys from 'lodash/keys';
import flatten from 'lodash/flatten';
import { join, resolve as pResolve } from 'node:path';
import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities';
import { join } from 'node:path';
import { configHandler } from '@contentstack/cli-utilities';

import config from '../../config';
import { log, formatError, fsUtil, fileHelper } from '../../utils';
import BaseClass, { ApiOptions } from './base-class';
import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types';
Expand All @@ -21,17 +20,23 @@ type TaxonomyPayload = {

export default class ImportTaxonomies extends BaseClass {
private taxonomiesMapperDirPath: string;
private termsMapperDirPath: string;
private taxonomiesFolderPath: string;
private termsFolderPath: string;
private taxSuccessPath: string;
private taxFailsPath: string;
private taxonomiesConfig: TaxonomiesConfig;
private termsConfig: TermsConfig;
private taxonomies: Record<string, unknown>;
private taxonomiesSuccess: Record<string, unknown> = {};
private taxonomiesFailed: Record<string, unknown> = {};
private termsFolderPath: string;
private termsMapperDirPath: string;
private termsConfig: TermsConfig;
private termsSuccessPath: string;
private termsFailsPath: string;
private taxonomyPayload: TaxonomyPayload;
public taxonomiesSuccess: Record<string, unknown> = {};
public taxonomiesFailed: Record<string, unknown> = {};
public termsSuccess: Record<string, Record<string, unknown>> = {};
public termsFailed: Record<string, Record<string, unknown>> = {};
public terms: Record<string, unknown> = {};
public taxonomyUIDs: string[] = [];

constructor({ importConfig, stackAPIClient }: ModuleClassParams) {
super({ importConfig, stackAPIClient });
Expand All @@ -43,6 +48,8 @@ export default class ImportTaxonomies extends BaseClass {
this.termsFolderPath = join(this.taxonomiesFolderPath, this.termsConfig.dirName);
this.taxSuccessPath = join(this.taxonomiesMapperDirPath, 'success.json');
this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json');
this.termsSuccessPath = join(this.termsMapperDirPath, 'success.json');
this.termsFailsPath = join(this.termsMapperDirPath, 'fails.json');
this.taxonomyPayload = {
baseUrl: '',
url: '',
Expand Down Expand Up @@ -84,45 +91,52 @@ export default class ImportTaxonomies extends BaseClass {
await fsUtil.makeDirectory(this.taxonomiesMapperDirPath);
await fsUtil.makeDirectory(this.termsMapperDirPath);

//Step 3 import taxonomy and create success & failure file
await this.importTaxonomies();
// create terms related to respective taxonomy
this.createTaxonomySuccessAndFailedFile();

if (!fileHelper.fileExistsSync(this.termsFolderPath)) {
log(this.importConfig, `No such file or directory - '${this.taxonomiesFolderPath}'`, 'error');
log(this.importConfig, `No such file or directory - '${this.termsFolderPath}'`, 'error');
return;
}
//Step 4 import terms and create success & failure file
await this.importTerms();

if (this.taxonomiesSuccess !== undefined && !isEmpty(this.taxonomiesSuccess)) {
fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesSuccess);
}

if (this.taxonomiesFailed !== undefined &&!isEmpty(this.taxonomiesFailed)) {
fsUtil.writeFile(this.taxFailsPath, this.taxonomiesFailed);
}
this.createTermSuccessAndFailedFile();

log(this.importConfig, 'Taxonomies have been imported successfully!', 'success');
}

/**
* create taxonomy and enter success & failure related data into taxonomies mapper file
* @method importTaxonomies
* @async
* @returns {Promise<any>} Promise<any>
*/
async importTaxonomies(): Promise<any> {
if (this.taxonomies === undefined || isEmpty(this.taxonomies)) {
log(this.importConfig, 'No Taxonomies Found', 'info');
return;
}

const apiContent = values(this.taxonomies);
this.taxonomyUIDs = keys(this.taxonomies);

const onSuccess = ({
response: { data, status } = { data: null, status: null },
apiData: { uid, name } = { uid: null, name: '' },
}: any) => {
//NOTE - Temp code to handle error thru API. Will remove this once sdk is ready
if ([200, 201, 202].includes(status)) {
this.taxonomiesSuccess[uid] = data;
const { taxonomy } = data;
this.taxonomiesSuccess[taxonomy.uid] = taxonomy;
log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success');
} else {
let errorMsg;
if ([500, 503, 502].includes(status)) errorMsg = data?.message || data;
else errorMsg = data?.error_message;
if (errorMsg === undefined) {
errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors));
}
this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`;
log(this.importConfig, `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error');
}
Expand Down Expand Up @@ -152,7 +166,7 @@ export default class ImportTaxonomies extends BaseClass {
includeParamOnCompletion: true,
additionalInfo: this.taxonomyPayload,
},
concurrencyLimit: config.concurrency || config.fetchConcurrency || 1,
concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1,
},
undefined,
false,
Expand All @@ -176,61 +190,94 @@ export default class ImportTaxonomies extends BaseClass {
return apiOptions;
}

/**
* create taxonomies success and fail in (mapper/taxonomies)
* @method createTaxonomySuccessAndFailedFile
*/
createTaxonomySuccessAndFailedFile() {
if (this.taxonomiesSuccess !== undefined && !isEmpty(this.taxonomiesSuccess)) {
fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesSuccess);
}

if (this.taxonomiesFailed !== undefined && !isEmpty(this.taxonomiesFailed)) {
fsUtil.writeFile(this.taxFailsPath, this.taxonomiesFailed);
}
}

/**
* create terms and enter success & failure related data into terms mapper file
* @method importTerms
* @async
* @returns {Promise<any>} Promise<any>
*/
async importTerms(): Promise<any> {
if (this.taxonomies === undefined || isEmpty(this.taxonomies)) {
if (!this.taxonomyUIDs?.length) {
return;
}

const listOfTaxonomyUIDs = keys(this.taxonomies);

const onSuccess = ({
response,
apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: '' },
response: { data, status } = { data: null, status: null },
apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: null },
}: any) => {
this.taxonomiesSuccess[uid] = response;
log(this.importConfig, `Term '${name}' imported successfully`, 'success');
//NOTE - Temp code to handle error thru API. Will remove this once sdk is ready
if ([200, 201, 202].includes(status)) {
if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {};
const { term } = data;
this.termsSuccess[taxonomy_uid][term.uid] = term;
log(this.importConfig, `Term '${name}' imported successfully`, 'success');
} else {
if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {};
let errorMsg;
if ([500, 503, 502].includes(status)) errorMsg = data?.message || data;
else errorMsg = data?.error_message;
if (errorMsg === undefined) {
errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors));
}
this.termsFailed[taxonomy_uid][uid] = `Terms '${name}' failed to be import. ${JSON.stringify(errorMsg)}`;
log(this.importConfig, `Terms '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error');
}
};

const onReject = ({ error, apiData }: any) => {
const { uid, taxonomy_uid, name } = apiData;
if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {};
const err = error?.message ? JSON.parse(error.message) : error;
const { name } = apiData;
if (err?.errors?.name) {
log(this.importConfig, `Term '${name}' already exists`, 'info');
} else {
this.taxonomiesSuccess[apiData.uid] = apiData;
this.termsFailed[taxonomy_uid][apiData.uid] = apiData;
log(this.importConfig, `Term '${name}' failed to be import ${formatError(error)}`, 'error');
log(this.importConfig, error, 'error');
}
};

for (const taxUID of listOfTaxonomyUIDs) {
const terms = fsUtil.readFile(
for (const taxUID of this.taxonomyUIDs) {
//read terms from respective taxonomy
this.terms = fsUtil.readFile(
join(this.termsFolderPath, `${taxUID}-${this.termsConfig.fileName}`),
true,
) as Record<string, unknown>;
const dirPath = pResolve(this.termsMapperDirPath, `${taxUID}-terms`);
if (!fileHelper.fileExistsSync(dirPath)) {
await fsUtil.makeDirectory(dirPath);
}
//success and fail path for particular taxonomy uid
const apiContent = values(terms);
await this.makeConcurrentCall(
{
apiContent,
processName: 'import terms',
apiParams: {
serializeData: this.serializeTerms.bind(this),
reject: onReject,
resolve: onSuccess,
entity: 'create-terms',
includeParamOnCompletion: true,
additionalInfo: this.taxonomyPayload,

if (this.terms !== undefined && !isEmpty(this.terms)) {
const apiContent = values(this.terms);
await this.makeConcurrentCall(
{
apiContent,
processName: 'import terms',
apiParams: {
serializeData: this.serializeTerms.bind(this),
reject: onReject,
resolve: onSuccess,
entity: 'create-terms',
includeParamOnCompletion: true,
additionalInfo: this.taxonomyPayload,
},
concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1,
},
concurrencyLimit: config.concurrency || config.fetchConcurrency || 1,
},
undefined,
false,
);
undefined,
false,
);
}
}
}

Expand All @@ -244,4 +291,18 @@ export default class ImportTaxonomies extends BaseClass {
apiOptions.apiData = term;
return apiOptions;
}

/**
* create terms success and fail in (mapper/taxonomies/terms)
* @method createTermSuccessAndFailedFile
*/
createTermSuccessAndFailedFile() {
if (this.termsSuccess !== undefined && !isEmpty(this.termsSuccess)) {
fsUtil.writeFile(this.termsSuccessPath, this.termsSuccess);
}

if (this.termsFailed !== undefined && !isEmpty(this.termsFailed)) {
fsUtil.writeFile(this.termsFailsPath, this.termsFailed);
}
}
}
2 changes: 1 addition & 1 deletion packages/contentstack-import/src/utils/asset-helper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import Bluebird from 'bluebird';
import * as url from 'url';
import * as path from 'path';
import { ContentstackClient, managementSDKClient } from '@contentstack/cli-utilities';
import { ImportConfig } from 'src/types';
import { ImportConfig } from '../types';
const debug = require('debug')('util:requests');
let _ = require('lodash');
let { marked } = require('marked');
Expand Down

0 comments on commit 3344ff9

Please sign in to comment.