Skip to content

Commit

Permalink
Merge pull request #1068 from contentstack/feat/CS-41092
Browse files Browse the repository at this point in the history
Import content types and entries module supports taxonomy
  • Loading branch information
aman19K authored Oct 4, 2023
2 parents ecaf413 + 9f32468 commit 377e5ff
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 9 deletions.
11 changes: 8 additions & 3 deletions packages/contentstack-import/src/import/modules/content-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import * as path from 'path';
import { isEmpty, find, cloneDeep, map } from 'lodash';
import { fsUtil, log, formatError, schemaTemplate, lookupExtension } from '../../utils';
import { fsUtil, log, formatError, schemaTemplate, lookupExtension, lookUpTaxonomy } from '../../utils';
import { ImportConfig, ModuleClassParams } from '../../types';
import BaseClass, { ApiOptions } from './base-class';
import { updateFieldRules } from '../../utils/content-type-helper';
Expand Down Expand Up @@ -49,6 +49,8 @@ export default class ContentTypesImport extends BaseClass {
limit: number;
writeConcurrency?: number;
};
private taxonomiesPath: string;
public taxonomies: Record<string, unknown>;

constructor({ importConfig, stackAPIClient }: ModuleClassParams) {
super({ importConfig, stackAPIClient });
Expand All @@ -75,6 +77,7 @@ export default class ContentTypesImport extends BaseClass {
this.gFs = [];
this.createdGFs = [];
this.pendingGFs = [];
this.taxonomiesPath = path.join(importConfig.data, 'mapper/taxonomies', 'success.json');
}

async start(): Promise<any> {
Expand All @@ -85,7 +88,6 @@ export default class ContentTypesImport extends BaseClass {
* Update pending global fields
* write field rules
*/

this.cTs = fsUtil.readFile(path.join(this.cTsFolderPath, 'schema.json')) as Record<string, unknown>[];
if (!this.cTs || isEmpty(this.cTs)) {
log(this.importConfig, 'No content type found to import', 'info');
Expand All @@ -95,6 +97,7 @@ export default class ContentTypesImport extends BaseClass {
this.installedExtensions = (
((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} }
).extension_uid;
this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record<string, unknown>;

await this.seedCTs();
log(this.importConfig, 'Created content types', 'success');
Expand Down Expand Up @@ -152,7 +155,7 @@ export default class ContentTypesImport extends BaseClass {

async updateCTs(): Promise<any> {
const onSuccess = ({ response: contentType, apiData: { uid } }: any) => {
log(this.importConfig, `${uid} updated with references`, 'success');
log(this.importConfig, `'${uid}' updated with references`, 'success');
};
const onReject = ({ error, apiData: { uid } }: any) => {
log(this.importConfig, formatError(error), 'error');
Expand Down Expand Up @@ -186,6 +189,8 @@ export default class ContentTypesImport extends BaseClass {
}
this.fieldRules.push(contentType.uid);
}
//will remove taxonomy if taxonomy doesn't exists in stack
lookUpTaxonomy(contentType.schema, this.taxonomies);
lookupExtension(
this.importConfig,
contentType.schema,
Expand Down
8 changes: 8 additions & 0 deletions packages/contentstack-import/src/import/modules/entries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
lookupEntries,
lookupAssets,
fileHelper,
lookUpTerms,
} from '../../utils';
import { ModuleClassParams } from '../../types';
import BaseClass, { ApiOptions } from './base-class';
Expand Down Expand Up @@ -53,6 +54,8 @@ export default class EntriesImport extends BaseClass {
private entriesUidMapper: Record<string, any>;
private envs: Record<string, any>;
private autoCreatedEntries: Record<string, any>[];
private taxonomiesPath: string;
public taxonomies: Record<string, unknown>;

constructor({ importConfig, stackAPIClient }: ModuleClassParams) {
super({ importConfig, stackAPIClient });
Expand All @@ -64,6 +67,7 @@ export default class EntriesImport extends BaseClass {
this.uniqueUidMapperPath = path.join(this.entriesMapperPath, 'unique-mapping.json');
this.modifiedCTsPath = path.join(this.entriesMapperPath, 'modified-schemas.json');
this.marketplaceAppMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json');
this.taxonomiesPath = path.join(this.importConfig.data, 'mapper', 'taxonomies', 'terms', 'success.json');
this.entriesConfig = importConfig.modules.entries;
this.entriesPath = path.resolve(importConfig.data, this.entriesConfig.dirName);
this.cTsPath = path.resolve(importConfig.data, importConfig.modules['content-types'].dirName);
Expand Down Expand Up @@ -96,6 +100,8 @@ export default class EntriesImport extends BaseClass {

this.assetUidMapper = (fsUtil.readFile(this.assetUidMapperPath) as Record<string, any>) || {};
this.assetUrlMapper = (fsUtil.readFile(this.assetUrlMapperPath) as Record<string, any>) || {};

this.taxonomies = (fsUtil.readFile(this.taxonomiesPath) as Record<string, any>);

fsUtil.makeDirectory(this.entriesMapperPath);
await this.disableMandatoryCTReferences();
Expand Down Expand Up @@ -360,6 +366,8 @@ export default class EntriesImport extends BaseClass {
if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) {
entry = removeEntryRefsFromJSONRTE(entry, contentType.schema);
}
//will remove term if term doesn't exists in taxonomy
lookUpTerms(contentType?.schema, entry, this.taxonomies);
// will replace all old asset uid/urls with new ones
entry = lookupAssets(
{
Expand Down
13 changes: 7 additions & 6 deletions packages/contentstack-import/src/import/modules/taxonomies.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import keys from 'lodash/keys';
import pick from 'lodash/pick';
import { join } from 'node:path';
import values from 'lodash/values';
import isEmpty from 'lodash/isEmpty';
Expand All @@ -13,7 +14,7 @@ import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types';
type TaxonomyPayload = {
baseUrl: string;
url: string;
mgToken: string;
mgToken?: string;
reqPayload: Record<string, unknown>;
headers: Record<string, unknown>;
};
Expand Down Expand Up @@ -118,7 +119,7 @@ export default class ImportTaxonomies extends BaseClass {
return;
}

const apiContent = values(this.taxonomies);
const apiContent = values(this.taxonomies) as Record<string, any>[];;
this.taxonomyUIDs = keys(this.taxonomies);

const onSuccess = ({
Expand All @@ -128,10 +129,10 @@ export default class ImportTaxonomies extends BaseClass {
//NOTE - Temp code to handle error thru API. Will remove this once sdk is ready
if ([200, 201, 202].includes(status)) {
const { taxonomy } = data;
this.taxonomiesSuccess[taxonomy.uid] = taxonomy;
this.taxonomiesSuccess[taxonomy.uid] = pick(taxonomy, ['name', 'description']);
log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success');
} else {
let errorMsg;
let errorMsg:any;
if ([500, 503, 502].includes(status)) errorMsg = data?.message || data;
else errorMsg = data?.error_message;
if (errorMsg === undefined) {
Expand Down Expand Up @@ -223,7 +224,7 @@ export default class ImportTaxonomies extends BaseClass {
if ([200, 201, 202].includes(status)) {
if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {};
const { term } = data;
this.termsSuccess[taxonomy_uid][term.uid] = term;
this.termsSuccess[taxonomy_uid][term.uid] = pick(term, ['name']);
log(this.importConfig, `Term '${name}' imported successfully`, 'success');
} else {
if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {};
Expand Down Expand Up @@ -259,7 +260,7 @@ export default class ImportTaxonomies extends BaseClass {
) as Record<string, unknown>;

if (this.terms !== undefined && !isEmpty(this.terms)) {
const apiContent = values(this.terms);
const apiContent = values(this.terms) as Record<string, any>[];
await this.makeConcurrentCall(
{
apiContent,
Expand Down
1 change: 1 addition & 0 deletions packages/contentstack-import/src/utils/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,4 @@ export {
restoreJsonRteEntryRefs,
} from './entries-helper';
export * from './common-helper';
export { lookUpTaxonomy, lookUpTerms } from './taxonomies-helper';
114 changes: 114 additions & 0 deletions packages/contentstack-import/src/utils/taxonomies-helper.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/**
* taxonomy lookup
*/
import { cliux } from '@contentstack/cli-utilities';

/**
* check and remove if referenced taxonomy doesn't exists in stack
* @param {any} schema content type schema
* @param {Record<string, unknown>} taxonomies created taxonomies
*/
export const lookUpTaxonomy = function (schema: any, taxonomies: Record<string, unknown>) {
for (let i in schema) {
if (schema[i].data_type === 'taxonomy') {
const taxonomyFieldData = schema[i].taxonomies as Record<string, any>[];
const { updatedTaxonomyData, isTaxonomyFieldRemoved } = verifyAndRemoveTaxonomy(taxonomyFieldData, taxonomies);

//Handle API error -> The 'taxonomies' property must have atleast one taxonomy object. Remove taxonomy field from schema.
if (isTaxonomyFieldRemoved) {
schema.splice(i, 1);
} else {
schema[i].taxonomies = updatedTaxonomyData;
}
}
}
};

/**
* verify and remove referenced taxonomy with warning from respective content type
* @param {Record<string, any>[]} taxonomyFieldData
* @param {Record<string, unknown>} taxonomies created taxonomies
* @returns
*/
const verifyAndRemoveTaxonomy = function (
taxonomyFieldData: Record<string, any>[],
taxonomies: Record<string, unknown>,
): {
updatedTaxonomyData: Record<string, any>[];
isTaxonomyFieldRemoved: boolean;
} {
let isTaxonomyFieldRemoved: boolean = false;

for (let index = 0; index < taxonomyFieldData?.length; index++) {
const taxonomyData = taxonomyFieldData[index];

if (taxonomies === undefined || !taxonomies.hasOwnProperty(taxonomyData?.taxonomy_uid)) {
// remove taxonomy from taxonomies field data with warning if respective taxonomy doesn't exists
cliux.print(`Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from taxonomies field`, {
color: 'yellow',
});
taxonomyFieldData.splice(index, 1);
--index;
}
}

if (!taxonomyFieldData?.length) {
cliux.print('Taxonomy does not exist. Removing the field from content type', { color: 'yellow' });
isTaxonomyFieldRemoved = true;
}

return {
updatedTaxonomyData: taxonomyFieldData,
isTaxonomyFieldRemoved,
};
};

/**
* check and remove if referenced terms doesn't exists in taxonomy
* @param {Record<string, any>[]} ctSchema content type schema
* @param {any} entry
* @param {Record<string, any>} taxonomiesAndTermData created taxonomies and terms
*/
export const lookUpTerms = function (
ctSchema: Record<string, any>[],
entry: any,
taxonomiesAndTermData: Record<string, any>,
) {
for (let index = 0; index < ctSchema?.length; index++) {
if (ctSchema[index].data_type === 'taxonomy') {
const taxonomyFieldData = entry[ctSchema[index].uid];
const updatedTaxonomyData = verifyAndRemoveTerms(taxonomyFieldData, taxonomiesAndTermData);
entry[ctSchema[index].uid] = updatedTaxonomyData;
}
}
};

/**
* verify and remove referenced term with warning from respective entry
* @param {Record<string, any>[]} taxonomyFieldData entry taxonomies data
* @param {Record<string, any>} taxonomiesAndTermData created taxonomies and terms
* @returns { Record<string, any>[]}
*/
const verifyAndRemoveTerms = function (
taxonomyFieldData: Record<string, any>[],
taxonomiesAndTermData: Record<string, any>,
): Record<string, any>[] {
for (let index = 0; index < taxonomyFieldData?.length; index++) {
const taxonomyData = taxonomyFieldData[index];
const taxUID = taxonomyData?.taxonomy_uid;
const termUID = taxonomyData?.term_uid;

if (
taxonomiesAndTermData === undefined ||
!taxonomiesAndTermData.hasOwnProperty(taxUID) ||
(taxonomiesAndTermData.hasOwnProperty(taxUID) && !taxonomiesAndTermData[taxUID].hasOwnProperty(termUID))
) {
// remove term from taxonomies field data with warning if respective term doesn't exists
cliux.print(`Term '${termUID}' does not exist. Removing it from taxonomy - '${taxUID}'`, { color: 'yellow' });
taxonomyFieldData.splice(index, 1);
--index;
}
}

return taxonomyFieldData;
};

0 comments on commit 377e5ff

Please sign in to comment.