From 7ca00a2c1242331ea0d5cf8948ba41fd14e128a7 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Wed, 2 Aug 2023 17:29:13 +0530 Subject: [PATCH 1/5] entries import --- packages/contentstack-export/package.json | 4 +- .../contentstack-export/src/config/index.ts | 2 +- .../src/export/modules/entries.ts | 2 +- packages/contentstack-export/tsconfig.json | 2 +- packages/contentstack-import/package.json | 8 +- .../contentstack-import/src/config/index.ts | 2 +- .../src/import/modules/base-class.ts | 16 +- .../src/import/modules/entries.ts | 311 ++++++++++++++++++ .../src/utils/asset-helper.ts | 10 +- .../src/utils/entries-helper.ts | 182 ++++++++++ .../contentstack-import/src/utils/index.ts | 4 +- packages/contentstack-import/tsconfig.json | 2 +- 12 files changed, 524 insertions(+), 21 deletions(-) create mode 100644 packages/contentstack-import/src/import/modules/entries.ts diff --git a/packages/contentstack-export/package.json b/packages/contentstack-export/package.json index e9a1149102..bfd320b7ee 100644 --- a/packages/contentstack-export/package.json +++ b/packages/contentstack-export/package.json @@ -80,9 +80,9 @@ "plugin" ], "license": "MIT", - "main": "./lib/commands/cm/stacks/export.js", + "main": "./src/commands/cm/stacks/export.js", "oclif": { - "commands": "./lib/commands", + "commands": "./src/commands", "bin": "csdx", "devPlugins": [ "@oclif/plugin-help" diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 00ab77a641..b35863519d 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -137,7 +137,7 @@ const config: DefaultConfig = { // total no of entries fetched in each content type in a single call limit: 100, dependencies: ['locales', 'content-types'], - exportVersions: true, + exportVersions: false, }, extensions: { dirName: 'extensions', diff --git a/packages/contentstack-export/src/export/modules/entries.ts b/packages/contentstack-export/src/export/modules/entries.ts index 15557cbbdd..f795cee78c 100644 --- a/packages/contentstack-export/src/export/modules/entries.ts +++ b/packages/contentstack-export/src/export/modules/entries.ts @@ -56,7 +56,7 @@ export default class EntriesExport extends BaseClass { for (let entryRequestOption of entryRequestOptions) { log( this.exportConfig, - `Starting export of entries of contenttype - ${entryRequestOption.contentType} locale - ${entryRequestOption.locale}`, + `Starting export of entries of content type - ${entryRequestOption.contentType} locale - ${entryRequestOption.locale}`, 'info', ); await this.getEntries(entryRequestOption); diff --git a/packages/contentstack-export/tsconfig.json b/packages/contentstack-export/tsconfig.json index 5136623392..da8fc4b724 100644 --- a/packages/contentstack-export/tsconfig.json +++ b/packages/contentstack-export/tsconfig.json @@ -12,7 +12,7 @@ "skipLibCheck": true, "sourceMap": false, "esModuleInterop": true, - "noImplicitAny": true, + "noImplicitAny": false, "lib": [ "ES2019", "es2020.promise" diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index f9d6c1097e..8a247c7755 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -7,7 +7,7 @@ "dependencies": { "@contentstack/cli-command": "^1.2.11", "@contentstack/cli-utilities": "^1.5.1", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.0", "@oclif/config": "^1.18.3", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", @@ -79,10 +79,10 @@ "cli", "plugin" ], - "main": "./lib/commands/cm/stacks/import.js", + "main": "./src/commands/cm/stacks/import.js", "license": "MIT", "oclif": { - "commands": "./lib/commands", + "commands": "./src/commands", "bin": "csdx", "repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-import/<%- commandPath %>" }, @@ -96,4 +96,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} +} \ No newline at end of file diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 89da6a35cc..8915665086 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -388,7 +388,7 @@ const config: DefaultConfig = { developerHubBaseUrl: '', marketplaceAppEncryptionKey: 'nF2ejRQcTv', getEncryptionKeyMaxRetry: 3, - useNewModuleStructure: true + useNewModuleStructure: true, // useBackedupDir: '', // backupConcurrency: 10, }; diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index d7e303f235..374e79c5a5 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -43,7 +43,8 @@ export type ApiModuleType = | 'update-labels' | 'create-webhooks' | 'create-workflows' - | 'create-custom-role'; + | 'create-custom-role' + | 'create-entries'; export type ApiOptions = { uid?: string; @@ -231,7 +232,7 @@ export default abstract class BaseClass { apiOptions = apiOptions.serializeData(apiOptions); } - const { uid, entity, reject, resolve, apiData, additionalInfo, includeParamOnCompletion } = apiOptions; + const { uid, entity, reject, resolve, apiData, additionalInfo = {}, includeParamOnCompletion } = apiOptions; const onSuccess = (response: any) => resolve({ @@ -295,6 +296,9 @@ export default abstract class BaseClass { case 'create-cts': return this.stack.contentType().create(apiData).then(onSuccess).catch(onReject); case 'update-cts': + if (additionalInfo.skip) { + return Promise.resolve(onSuccess(apiData)); + } return apiData.update().then(onSuccess).catch(onReject); case 'update-gfs': return apiData.update().then(onSuccess).catch(onReject); @@ -337,7 +341,13 @@ export default abstract class BaseClass { .create({ role: apiData as RoleData }) .then(onSuccess) .catch(onReject); - + case 'create-entries': + return this.stack + .contentType(additionalInfo.ctUid) + .entry() + .create({ entry: apiData }) + .then(onSuccess) + .catch(onReject); default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts new file mode 100644 index 0000000000..7bed6a68c1 --- /dev/null +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -0,0 +1,311 @@ +/* eslint-disable no-prototype-builtins */ +/*! + * Contentstack Import + * Copyright (c) 2019 Contentstack LLC + * MIT Licensed + */ + +import * as path from 'path'; +import { isEmpty, values, cloneDeep } from 'lodash'; +import { FsUtility } from '@contentstack/cli-utilities'; +import { + fsUtil, + log, + formatError, + lookupExtension, + suppressSchemaReference, + removeUidsFromJsonRteFields, + removeEntryRefsFromJSONRTE, + lookupAssets, +} from '../../utils'; +import { ModuleClassParams } from '../../types'; +import BaseClass, { ApiOptions } from './base-class'; + +export default class EntriesImport extends BaseClass { + private assetUidMapperPath: string; + private assetUidMapper: Record; + private assetUrlMapper: Record; + private assetUrlMapperPath: string; + private entriesMapperPath: string; + private envPath: string; + private entriesUIDMapperPath: string; + private uniqueUidMapperPath: string; + private modifiedCTsPath: string; + private marketplaceAppMapperPath: string; + private entriesConfig: Record; + private cTsPath: string; + private localesPath: string; + private importConcurrency: number; + private entriesPath: string; + private cTs: Record[]; + private modifiedCTs: Record[]; + private refCTs: Record; + private jsonRteCTs: Record; + private jsonRteCTsWithRef: Record; + private jsonRteEntries: Record; + private installedExtensions: Record[]; + private createdEntries: Record[]; + private failedEntries: Record[]; + private locales: Record[]; + private entriesUidMapper: Record; + + constructor({ importConfig, stackAPIClient }: ModuleClassParams) { + super({ importConfig, stackAPIClient }); + this.assetUidMapperPath = path.resolve(importConfig.data, 'mapper', 'assets', 'uid-mapping.json'); + this.assetUrlMapperPath = path.resolve(importConfig.data, 'mapper', 'assets', 'url-mapping.json'); + this.entriesMapperPath = path.resolve(importConfig.data, 'mapper', 'entries'); + this.envPath = path.resolve(importConfig.data, 'environments', 'environments.json'); + this.entriesUIDMapperPath = path.join(this.entriesMapperPath, 'uid-mapping.json'); + this.uniqueUidMapperPath = path.join(this.entriesMapperPath, 'unique-mapping.json'); + this.modifiedCTsPath = path.join(this.entriesMapperPath, 'modified-schemas.json'); + this.marketplaceAppMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json'); + // this.createdEntriesWOUidPath = path.join(this.entryMapperPath, 'created-entries-wo-uid.json'); + // this.failedWOPath = path.join(this.entryMapperPath, 'failedWO.json'); + this.entriesConfig = importConfig.modules.entries; + this.entriesPath = path.resolve(importConfig.data, this.entriesConfig.dirName); + this.cTsPath = path.resolve(importConfig.data, importConfig.modules['content-types'].dirName); + this.localesPath = path.resolve( + importConfig.data, + importConfig.modules.locales.dirName, + importConfig.modules.locales.fileName, + ); + this.importConcurrency = this.entriesConfig.importConcurrency || importConfig.importConcurrency; + this.entriesUidMapper = {}; + } + + async start(): Promise { + try { + /** + * Read CTS + * Suppress CTS + * Read locales + * Create request objs + * create entries + * update entries + */ + + this.cTs = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record[]; + if (!this.cTs || isEmpty(this.cTs)) { + log(this.importConfig, 'No content type found', 'info'); + return; + } + this.installedExtensions = ( + ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} } + ).extension_uid; + + this.assetUidMapper = (fsUtil.readFile(this.assetUidMapperPath) as Record) || {}; + this.assetUrlMapper = (fsUtil.readFile(this.assetUrlMapperPath) as Record) || {}; + + await this.disableMandatoryCTReferences(); + fsUtil.makeDirectory(this.entriesMapperPath); + this.locales = fsUtil.readFile(this.localesPath) as Record[]; + const entryRequestOptions = this.populateEntryCreatePayload(); + for (let entryRequestOption of entryRequestOptions) { + log( + this.importConfig, + `Starting to create entries for ${entryRequestOption.cTUid} in locale ${entryRequestOption.locale}`, + 'info', + ); + await this.createEntries(entryRequestOption); + } + log(this.importConfig, 'Entries created successfully', 'info'); + } catch (error) { + log(this.importConfig, formatError(error), 'error'); + throw new Error('Error while importing entries'); + } + } + + async disableMandatoryCTReferences() { + const onSuccess = ({ response: contentType, apiData: { uid } }: any) => { + log(this.importConfig, `${uid} content type references removed temporarily`, 'success'); + }; + const onReject = ({ error, apiData: { uid } }: any) => { + log(this.importConfig, formatError(error), 'error'); + throw new Error(`${uid} content type references removal failed`); + }; + return await this.makeConcurrentCall({ + processName: 'Update content types (removing mandatory references temporarily)', + apiContent: this.cTs, + apiParams: { + serializeData: this.serializeUpdateCTs.bind(this), + reject: onReject.bind(this), + resolve: onSuccess.bind(this), + entity: 'update-cts', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.importConcurrency, + }).then(() => { + fsUtil.writeFile(this.modifiedCTsPath, this.modifiedCTs); + }); + } + + /** + * @method serializeUpdateCTs + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeUpdateCTs(apiOptions: ApiOptions): ApiOptions { + const { apiData: contentType } = apiOptions; + if (contentType.field_rules) { + delete contentType.field_rules; + } + const flag = { + suppressed: false, + references: false, + jsonRte: false, + jsonRteEmbeddedEntries: false, + }; + suppressSchemaReference(contentType.schema, flag); + // Check if suppress modified flag + if (flag.suppressed) { + this.modifiedCTs.push(this.cTs[contentType.uid]); + } else { + // Note: Skips the content type from update if no reference found + apiOptions.additionalInfo = { skip: true }; + return apiOptions; + } + + if (flag.references) { + this.refCTs.push(contentType.uid); + } + + if (flag.jsonRte) { + this.jsonRteCTs.push(contentType.uid); + if (flag.jsonRteEmbeddedEntries) { + this.jsonRteCTsWithRef.push(contentType.uid); + if (this.refCTs.indexOf(contentType.uid) === -1) { + this.refCTs.push(contentType.uid); + } + } + } + lookupExtension( + this.importConfig, + contentType.schema, + this.importConfig.preserveStackVersion, + this.installedExtensions, + ); + const contentTypePayload = this.stack.contentType(contentType.uid); + Object.assign(contentTypePayload, cloneDeep(contentType)); + apiOptions.apiData = contentTypePayload; + return apiOptions; + } + + populateEntryCreatePayload(): { cTUid: string; locale: string }[] { + const requestObjects: { cTUid: string; locale: string }[] = []; + this.cTs.forEach((contentType) => { + for (let locale in this.locales) { + requestObjects.push({ + cTUid: contentType.uid, + locale: this.locales[locale].code, + }); + } + requestObjects.push({ + cTUid: contentType.uid, + locale: this.importConfig.master_locale.code, + }); + }); + + return requestObjects; + } + + async createEntries({ cTUid, locale }: { cTUid: string; locale: string }): Promise { + const processName = 'Create Entries'; + const indexFileName = 'index.json'; + const basePath = path.join(this.entriesPath, cTUid, locale); + const fs = new FsUtility({ basePath, indexFileName }); + const indexer = fs.indexFileContent; + const indexerCount = values(indexer).length; + // Write created entries + const entriesCreateFileHelper = new FsUtility({ + moduleName: 'created-entries', + indexFileName: 'index.json', + basePath: path.join(this.entriesMapperPath, cTUid, locale), + chunkFileSize: this.entriesConfig.chunkFileSize, + keepMetadata: false, + omitKeys: this.entriesConfig.invalidKeys, + }); + + const onSuccess = ({ response, apiData: { uid, url, title } }: any) => { + log(this.importConfig, `Created entry: '${title}' of content type ${cTUid} in locale ${locale}`, 'info'); + this.entriesUidMapper[uid] = response.uid; + entriesCreateFileHelper.writeIntoFile({ [uid]: response } as any, { mapKeyVal: true }); + }; + const onReject = ({ error, apiData: { uid, title } }: any) => { + log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to create`, 'error'); + log(this.importConfig, formatError(error), 'error'); + this.failedEntries.push({ content_type: cTUid, locale, entry: { uid, title } }); + }; + + for (const index in indexer) { + const chunk = await fs.readChunkFiles.next().catch((error) => { + log(this.importConfig, formatError(error), 'error'); + }); + + if (chunk) { + let apiContent = values(chunk as Record[]); + await this.makeConcurrentCall({ + apiContent, + processName, + indexerCount, + currentIndexer: +index, + apiParams: { + reject: onReject, + resolve: onSuccess, + entity: 'create-entries', + includeParamOnCompletion: true, + serializeData: this.serializeEntries.bind(this), + additionalInfo: { cTUid, locale }, + }, + concurrencyLimit: this.importConcurrency, + }).then(() => { + fs?.completeFile(true); + log(this.importConfig, `Created entries for content type ${cTUid} in locale ${locale}`, 'success'); + }); + } + } + } + + /** + * @method serializeEntries + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeEntries(apiOptions: ApiOptions): ApiOptions { + let { + apiData: entry, + additionalInfo: { cTUid, locale }, + } = apiOptions; + + if (this.jsonRteCTs.indexOf(cTUid) > -1) { + entry = removeUidsFromJsonRteFields(entry, this.cTs[cTUid].schema); + } + // remove entry references from json-rte fields + if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) { + entry = removeEntryRefsFromJSONRTE(entry, this.cTs[cTUid].schema); + } + // will replace all old asset uid/urls with new ones + entry = lookupAssets( + { + content_type: this.cTs[cTUid], + entry: entry, + }, + this.assetUidMapper, + this.assetUrlMapper, + path.join(this.entriesPath, cTUid, locale), + this.installedExtensions, + ); + delete entry.publish_details; + apiOptions.apiData = entry; + return apiOptions; + } + + async updateEntriesWithReferences(): Promise {} + + async enableMandatoryCTReferences(): Promise {} + + async removeAutoCreatedEntries(): Promise {} + + async updateFieldRules(): Promise {} + + async publishEntries(): Promise {} +} diff --git a/packages/contentstack-import/src/utils/asset-helper.ts b/packages/contentstack-import/src/utils/asset-helper.ts index 0832824d88..c8fa8cb1ea 100644 --- a/packages/contentstack-import/src/utils/asset-helper.ts +++ b/packages/contentstack-import/src/utils/asset-helper.ts @@ -53,11 +53,11 @@ export const uploadAssetHelper = function (config: ImportConfig, req: any, fsPat // get assets object export const lookupAssets = function ( - data: any, - mappedAssetUids: string[], - mappedAssetUrls: string[], - assetUidMapperPath: string[], - installedExtensions: string[], + data: Record, + mappedAssetUids: Record, + mappedAssetUrls: Record, + assetUidMapperPath: string, + installedExtensions: Record[], ) { if ( !_.has(data, 'entry') || diff --git a/packages/contentstack-import/src/utils/entries-helper.ts b/packages/contentstack-import/src/utils/entries-helper.ts index e432fe01eb..f869058b3f 100644 --- a/packages/contentstack-import/src/utils/entries-helper.ts +++ b/packages/contentstack-import/src/utils/entries-helper.ts @@ -244,3 +244,185 @@ function findUidsInNewRefFields(entry: any, uids: string[]) { } } } + +export const removeUidsFromJsonRteFields = ( + entry: Record, + ctSchema: Record[], +): Record => { + for (const element of ctSchema) { + switch (element.data_type) { + case 'blocks': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e: any) => { + let key = Object.keys(e).pop(); + let subBlock = element.blocks.filter((block: any) => block.uid === key).pop(); + e[key] = removeUidsFromJsonRteFields(e[key], subBlock.schema); + return e; + }); + } + } + break; + } + case 'global_field': + case 'group': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e) => { + e = removeUidsFromJsonRteFields(e, element.schema); + return e; + }); + } else { + entry[element.uid] = removeUidsFromJsonRteFields(entry[element.uid], element.schema); + } + } + break; + } + case 'json': { + if (entry[element.uid] && element.field_metadata.rich_text_type) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((jsonRteData: any) => { + delete jsonRteData.uid; // remove uid + + if (_.isObject(jsonRteData.attrs)) { + jsonRteData.attrs.dirty = true; + } + + if (!_.isEmpty(jsonRteData.children)) { + jsonRteData.children = _.map(jsonRteData.children, (child) => removeUidsFromChildren(child)); + } + + return jsonRteData; + }); + } else { + delete entry[element.uid].uid; // remove uid + if (entry[element.uid] && _.isObject(entry[element.uid].attrs)) { + entry[element.uid].attrs.dirty = true; + } + if (entry[element.uid] && !_.isEmpty(entry[element.uid].children)) { + entry[element.uid].children = _.map(entry[element.uid].children, (child) => + removeUidsFromChildren(child), + ); + } + } + } + break; + } + } + } + return entry; +}; + +function removeUidsFromChildren(children: Record[] | any) { + if (children.length && children.length > 0) { + return children.map((child: any) => { + if (child.type && child.type.length > 0) { + delete child.uid; // remove uid + + if (_.isObject(child.attrs)) { + child.attrs.dirty = true; + } + } + if (child.children && child.children.length > 0) { + child.children = removeUidsFromChildren(child.children); + } + return child; + }); + } else { + if (children.type && children.type.length > 0) { + delete children.uid; // remove uid + if (_.isObject(children.attrs)) { + children.attrs.dirty = true; + } + } + if (children.children && children.children.length > 0) { + children.children = removeUidsFromChildren(children.children); + } + return children; + } +} + +export const removeEntryRefsFromJSONRTE = (entry: Record, ctSchema: Record[]) => { + for (const element of ctSchema) { + switch (element.data_type) { + case 'blocks': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e: any) => { + let key = Object.keys(e).pop(); + let subBlock = element.blocks.filter((block: any) => block.uid === key).pop(); + e[key] = removeEntryRefsFromJSONRTE(e[key], subBlock.schema); + return e; + }); + } + } + break; + } + case 'global_field': + case 'group': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e) => { + e = removeEntryRefsFromJSONRTE(e, element.schema); + return e; + }); + } else { + entry[element.uid] = removeEntryRefsFromJSONRTE(entry[element.uid], element.schema); + } + } + break; + } + case 'json': { + if (entry[element.uid] && element.field_metadata.rich_text_type) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((jsonRteData: any) => { + // repeated code from else block, will abstract later + let entryReferences = jsonRteData.children.filter((e: any) => doEntryReferencesExist(e)); + if (entryReferences.length > 0) { + jsonRteData.children = jsonRteData.children.filter((e: any) => !doEntryReferencesExist(e)); + return jsonRteData; // return jsonRteData without entry references + } else { + return jsonRteData; // return jsonRteData as it is, because there are no entry references + } + }); + } else { + let entryReferences = entry[element.uid].children.filter((e: any) => doEntryReferencesExist(e)); + if (entryReferences.length > 0) { + entry[element.uid].children = entry[element.uid].children.filter((e: any) => !doEntryReferencesExist(e)); + } + } + } + break; + } + } + } + return entry; +}; + +function doEntryReferencesExist(element: Record[] | any): boolean { + // checks if the children of p element contain any references + // only checking one level deep, not recursive + + if (element.length) { + for (const item of element) { + if ((item.type === 'p' || item.type === 'a') && item.children && item.children.length > 0) { + return doEntryReferencesExist(item.children); + } else if (isEntryRef(item)) { + return true; + } + } + } else { + if (isEntryRef(element)) { + return true; + } + + if ((element.type === 'p' || element.type === 'a') && element.children && element.children.length > 0) { + return doEntryReferencesExist(element.children); + } + } + return false; +} + +function isEntryRef(element: any) { + return element.type === 'reference' && element.attrs.type === 'entry'; +} diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index 138e589111..f794713082 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -16,9 +16,9 @@ export { confirmToCloseProcess, getAllStackSpecificApps, ifAppAlreadyExist, - updateAppConfig + updateAppConfig, } from './marketplace-app-helper'; export { schemaTemplate, suppressSchemaReference, removeReferenceFields } from './content-type-helper'; export { lookupExtension } from './extension-helper'; -export { lookupEntries } from './entries-helper'; +export { lookupEntries, removeUidsFromJsonRteFields, removeEntryRefsFromJSONRTE } from './entries-helper'; export * from './common-helper'; diff --git a/packages/contentstack-import/tsconfig.json b/packages/contentstack-import/tsconfig.json index 679afbeaf5..44beaaf137 100644 --- a/packages/contentstack-import/tsconfig.json +++ b/packages/contentstack-import/tsconfig.json @@ -11,7 +11,7 @@ "skipLibCheck": true, "sourceMap": false, "esModuleInterop": true, - "noImplicitAny": true, + "noImplicitAny": false, "lib": [ "es2019", "es2020.promise", From 6b108de08c40bdf816557a545f73b53a33dcc1bf Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Thu, 3 Aug 2023 15:26:45 +0530 Subject: [PATCH 2/5] added entries import --- packages/contentstack-export/package.json | 4 +- packages/contentstack-import/package.json | 4 +- .../contentstack-import/src/config/index.ts | 3 +- .../src/import/modules/base-class.ts | 18 +- .../src/import/modules/entries.ts | 416 ++++++++++++++++-- .../src/utils/entries-helper.ts | 159 ++++++- .../contentstack-import/src/utils/index.ts | 7 +- 7 files changed, 564 insertions(+), 47 deletions(-) diff --git a/packages/contentstack-export/package.json b/packages/contentstack-export/package.json index bfd320b7ee..e9a1149102 100644 --- a/packages/contentstack-export/package.json +++ b/packages/contentstack-export/package.json @@ -80,9 +80,9 @@ "plugin" ], "license": "MIT", - "main": "./src/commands/cm/stacks/export.js", + "main": "./lib/commands/cm/stacks/export.js", "oclif": { - "commands": "./src/commands", + "commands": "./lib/commands", "bin": "csdx", "devPlugins": [ "@oclif/plugin-help" diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index 8a247c7755..849e6c215c 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -79,10 +79,10 @@ "cli", "plugin" ], - "main": "./src/commands/cm/stacks/import.js", + "main": "./lib/commands/cm/stacks/import.js", "license": "MIT", "oclif": { - "commands": "./src/commands", + "commands": "./lib/commands", "bin": "csdx", "repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-import/<%- commandPath %>" }, diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 4cf99aca40..4a5621fd05 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -377,7 +377,8 @@ const config: DefaultConfig = { 'content-types', 'webhooks', 'custom-roles', - 'workflows' + 'workflows', + 'entries', ], rateLimit: 5, preserveStackVersion: false, diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 374e79c5a5..167d678148 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -44,7 +44,9 @@ export type ApiModuleType = | 'create-webhooks' | 'create-workflows' | 'create-custom-role' - | 'create-entries'; + | 'create-entries' + | 'update-entries' + | 'publish-entries'; export type ApiOptions = { uid?: string; @@ -343,11 +345,23 @@ export default abstract class BaseClass { .catch(onReject); case 'create-entries': return this.stack - .contentType(additionalInfo.ctUid) + .contentType(additionalInfo.cTUid) .entry() .create({ entry: apiData }) .then(onSuccess) .catch(onReject); + case 'update-entries': + return apiData.update().then(onSuccess).catch(onReject); + case 'publish-entries': + if (additionalInfo.skip) { + return Promise.resolve(onSuccess(apiData)); + } + return this.stack + .contentType(additionalInfo.cTUid) + .entry(additionalInfo.entryUid) + .publish({ publishDetails: apiData, locale: additionalInfo.locale }) + .then(onSuccess) + .catch(onReject); default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 7bed6a68c1..c87e1d8f96 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -6,8 +6,8 @@ */ import * as path from 'path'; -import { isEmpty, values, cloneDeep } from 'lodash'; -import { FsUtility } from '@contentstack/cli-utilities'; +import { isEmpty, values, cloneDeep, find, indexOf, forEach } from 'lodash'; +import { ContentType, FsUtility } from '@contentstack/cli-utilities'; import { fsUtil, log, @@ -16,7 +16,10 @@ import { suppressSchemaReference, removeUidsFromJsonRteFields, removeEntryRefsFromJSONRTE, + restoreJsonRteEntryRefs, + lookupEntries, lookupAssets, + fileHelper, } from '../../utils'; import { ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; @@ -39,7 +42,7 @@ export default class EntriesImport extends BaseClass { private entriesPath: string; private cTs: Record[]; private modifiedCTs: Record[]; - private refCTs: Record; + private refCTs: string[]; private jsonRteCTs: Record; private jsonRteCTsWithRef: Record; private jsonRteEntries: Record; @@ -48,6 +51,7 @@ export default class EntriesImport extends BaseClass { private failedEntries: Record[]; private locales: Record[]; private entriesUidMapper: Record; + private envs: Record; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -59,8 +63,6 @@ export default class EntriesImport extends BaseClass { this.uniqueUidMapperPath = path.join(this.entriesMapperPath, 'unique-mapping.json'); this.modifiedCTsPath = path.join(this.entriesMapperPath, 'modified-schemas.json'); this.marketplaceAppMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json'); - // this.createdEntriesWOUidPath = path.join(this.entryMapperPath, 'created-entries-wo-uid.json'); - // this.failedWOPath = path.join(this.entryMapperPath, 'failedWO.json'); this.entriesConfig = importConfig.modules.entries; this.entriesPath = path.resolve(importConfig.data, this.entriesConfig.dirName); this.cTsPath = path.resolve(importConfig.data, importConfig.modules['content-types'].dirName); @@ -71,6 +73,11 @@ export default class EntriesImport extends BaseClass { ); this.importConcurrency = this.entriesConfig.importConcurrency || importConfig.importConcurrency; this.entriesUidMapper = {}; + this.modifiedCTs = []; + this.refCTs = []; + this.jsonRteCTs = []; + this.jsonRteCTsWithRef = []; + this.envs = {}; } async start(): Promise { @@ -96,19 +103,72 @@ export default class EntriesImport extends BaseClass { this.assetUidMapper = (fsUtil.readFile(this.assetUidMapperPath) as Record) || {}; this.assetUrlMapper = (fsUtil.readFile(this.assetUrlMapperPath) as Record) || {}; - await this.disableMandatoryCTReferences(); fsUtil.makeDirectory(this.entriesMapperPath); - this.locales = fsUtil.readFile(this.localesPath) as Record[]; + await this.disableMandatoryCTReferences(); + this.locales = values(fsUtil.readFile(this.localesPath) as Record[]); + this.locales.unshift(this.importConfig.master_locale); // adds master locale to the list + + //Create Entries const entryRequestOptions = this.populateEntryCreatePayload(); for (let entryRequestOption of entryRequestOptions) { - log( - this.importConfig, - `Starting to create entries for ${entryRequestOption.cTUid} in locale ${entryRequestOption.locale}`, - 'info', - ); await this.createEntries(entryRequestOption); } - log(this.importConfig, 'Entries created successfully', 'info'); + await fileHelper.writeLargeFile(path.join(this.entriesMapperPath, 'uid-mapping.json'), this.entriesUidMapper); // TBD: manages mapper in one file, should find an alternative + fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); + + // Update entries with references + const entryUpdateRequestOptions = this.populateEntryUpdatePayload(); + for (let entryUpdateRequestOption of entryUpdateRequestOptions) { + await this.updateEntriesWithReferences(entryUpdateRequestOption).catch((error) => { + log( + this.importConfig, + `Error while updating entries references of ${entryUpdateRequestOption.cTUid} in locale ${entryUpdateRequestOption.locale}`, + 'error', + ); + log(this.importConfig, formatError(error), 'error'); + }); + } + fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); + + log(this.importConfig, 'Restoring content type changes', 'info'); + await this.enableMandatoryCTReferences().catch((error) => { + log(this.importConfig, `failed update content type references ${formatError(error)}`, 'error'); + }); + + log(this.importConfig, 'Removing entries from master language which got created by default', 'info'); + await this.removeAutoCreatedEntries().catch((error) => { + log(this.importConfig, `failed to remove auto created entries in master locale ${formatError(error)}`, 'error'); + }); + + // Update field rule of content types which are got removed earlier + log(this.importConfig, 'Updating the field rules of content type', 'info'); + await this.updateFieldRules().catch((error) => { + log(this.importConfig, `Error while updating field rules of content type ${formatError(error)}`, 'error'); + }); + log(this.importConfig, 'Entries imported successfully', 'success'); + + // Publishing entries + if (this.importConfig.entriesPublish) { + log(this.importConfig, 'Publishing entries', 'info'); + this.envs = fileHelper.readFileSync(this.envPath); + for (let entryRequestOption of entryRequestOptions) { + log( + this.importConfig, + `Starting publish entries for ${entryRequestOption.cTUid} in locale ${entryRequestOption.locale}`, + 'info', + ); + await this.publishEntries(entryRequestOption).catch((error) => { + log( + this.importConfig, + `Error in publishing entries of ${entryRequestOption.cTUid} in locale ${ + entryRequestOption.locale + } ${formatError(error)}`, + 'error', + ); + }); + } + log(this.importConfig, 'All the entries have been published successfully', 'success'); + } } catch (error) { log(this.importConfig, formatError(error), 'error'); throw new Error('Error while importing entries'); @@ -158,7 +218,7 @@ export default class EntriesImport extends BaseClass { suppressSchemaReference(contentType.schema, flag); // Check if suppress modified flag if (flag.suppressed) { - this.modifiedCTs.push(this.cTs[contentType.uid]); + this.modifiedCTs.push(find(this.cTs, { uid: contentType.uid })); } else { // Note: Skips the content type from update if no reference found apiOptions.additionalInfo = { skip: true }; @@ -191,21 +251,16 @@ export default class EntriesImport extends BaseClass { } populateEntryCreatePayload(): { cTUid: string; locale: string }[] { - const requestObjects: { cTUid: string; locale: string }[] = []; - this.cTs.forEach((contentType) => { - for (let locale in this.locales) { - requestObjects.push({ + const requestOptions: { cTUid: string; locale: string }[] = []; + for (let locale of this.locales) { + for (let contentType of this.cTs) { + requestOptions.push({ cTUid: contentType.uid, - locale: this.locales[locale].code, + locale: locale.code, }); } - requestObjects.push({ - cTUid: contentType.uid, - locale: this.importConfig.master_locale.code, - }); - }); - - return requestObjects; + } + return requestOptions; } async createEntries({ cTUid, locale }: { cTUid: string; locale: string }): Promise { @@ -215,6 +270,10 @@ export default class EntriesImport extends BaseClass { const fs = new FsUtility({ basePath, indexFileName }); const indexer = fs.indexFileContent; const indexerCount = values(indexer).length; + if (indexerCount === 0) { + return Promise.resolve(); + } + log(this.importConfig, `Starting to create entries for ${cTUid} in locale ${locale}`, 'info'); // Write created entries const entriesCreateFileHelper = new FsUtility({ moduleName: 'created-entries', @@ -224,11 +283,14 @@ export default class EntriesImport extends BaseClass { keepMetadata: false, omitKeys: this.entriesConfig.invalidKeys, }); + const contentType = find(this.cTs, { uid: cTUid }); - const onSuccess = ({ response, apiData: { uid, url, title } }: any) => { + const onSuccess = ({ response, apiData: { uid, url, title }, additionalInfo: { entryFileName } }: any) => { log(this.importConfig, `Created entry: '${title}' of content type ${cTUid} in locale ${locale}`, 'info'); this.entriesUidMapper[uid] = response.uid; - entriesCreateFileHelper.writeIntoFile({ [uid]: response } as any, { mapKeyVal: true }); + response.sourceEntryFilePath = path.join(basePath, entryFileName); // stores source file path temporarily + response.entryOldUid = uid; // stores old uid temporarily + entriesCreateFileHelper.writeIntoFile({ [response.uid]: response } as any, { mapKeyVal: true }); }; const onReject = ({ error, apiData: { uid, title } }: any) => { log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to create`, 'error'); @@ -254,11 +316,11 @@ export default class EntriesImport extends BaseClass { entity: 'create-entries', includeParamOnCompletion: true, serializeData: this.serializeEntries.bind(this), - additionalInfo: { cTUid, locale }, + additionalInfo: { contentType, locale, cTUid, entryFileName: indexer[index] }, }, concurrencyLimit: this.importConcurrency, }).then(() => { - fs?.completeFile(true); + entriesCreateFileHelper?.completeFile(true); log(this.importConfig, `Created entries for content type ${cTUid} in locale ${locale}`, 'success'); }); } @@ -273,20 +335,20 @@ export default class EntriesImport extends BaseClass { serializeEntries(apiOptions: ApiOptions): ApiOptions { let { apiData: entry, - additionalInfo: { cTUid, locale }, + additionalInfo: { cTUid, locale, contentType }, } = apiOptions; if (this.jsonRteCTs.indexOf(cTUid) > -1) { - entry = removeUidsFromJsonRteFields(entry, this.cTs[cTUid].schema); + entry = removeUidsFromJsonRteFields(entry, contentType.schema); } // remove entry references from json-rte fields if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) { - entry = removeEntryRefsFromJSONRTE(entry, this.cTs[cTUid].schema); + entry = removeEntryRefsFromJSONRTE(entry, contentType.schema); } // will replace all old asset uid/urls with new ones entry = lookupAssets( { - content_type: this.cTs[cTUid], + content_type: contentType, entry: entry, }, this.assetUidMapper, @@ -299,13 +361,291 @@ export default class EntriesImport extends BaseClass { return apiOptions; } - async updateEntriesWithReferences(): Promise {} + populateEntryUpdatePayload(): { cTUid: string; locale: string }[] { + const requestOptions: { cTUid: string; locale: string }[] = []; + for (let locale of this.locales) { + for (let cTUid of this.refCTs) { + requestOptions.push({ + cTUid, + locale: locale.code, + }); + } + } + return requestOptions; + } + + async updateEntriesWithReferences({ cTUid, locale }: { cTUid: string; locale: string }): Promise { + const processName = 'Update Entries'; + const indexFileName = 'index.json'; + const basePath = path.join(this.entriesMapperPath, cTUid, locale); + const fs = new FsUtility({ basePath, indexFileName }); + const indexer = fs.indexFileContent; + const indexerCount = values(indexer).length; + if (indexerCount === 0) { + return Promise.resolve(); + } + log(this.importConfig, `Starting to update entries with references for ${cTUid} in locale ${locale}`, 'info'); + + const contentType = find(this.cTs, { uid: cTUid }); + + const onSuccess = ({ response, apiData: { uid, url, title } }: any) => { + log(this.importConfig, `Updated entry: '${title}' of content type ${cTUid} in locale ${locale}`, 'info'); + }; + const onReject = ({ error, apiData: { uid, title } }: any) => { + log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to update`, 'error'); + log(this.importConfig, formatError(error), 'error'); + this.failedEntries.push({ content_type: cTUid, locale, entry: { uid, title } }); + }; + + for (const index in indexer) { + const chunk = await fs.readChunkFiles.next().catch((error) => { + log(this.importConfig, formatError(error), 'error'); + }); + + if (chunk) { + let apiContent = values(chunk as Record[]); + await this.makeConcurrentCall({ + apiContent, + processName, + indexerCount, + currentIndexer: +index, + apiParams: { + reject: onReject, + resolve: onSuccess, + entity: 'update-entries', + includeParamOnCompletion: true, + serializeData: this.serializeUpdateEntries.bind(this), + additionalInfo: { contentType, locale, cTUid }, + }, + concurrencyLimit: this.importConcurrency, + }).then(() => { + log(this.importConfig, `Updated entries for content type ${cTUid} in locale ${locale}`, 'success'); + }); + } + } + } + + /** + * @method serializeUpdateEntries + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeUpdateEntries(apiOptions: ApiOptions): ApiOptions { + let { + apiData: entry, + additionalInfo: { cTUid, locale, contentType }, + } = apiOptions; + + try { + const sourceEntryFilePath = entry.sourceEntryFilePath; + const sourceEntry = (fsUtil.readFile(sourceEntryFilePath) || {})[entry.entryOldUid]; + // Removing temp values + delete entry.sourceEntryFilePath; + delete entry.entryOldUid; + if (this.jsonRteCTs.indexOf(cTUid) > -1) { + // the entries stored in eSuccessFilePath, have the same uids as the entries from source data + entry = restoreJsonRteEntryRefs(entry, sourceEntry, contentType.schema, { + mappedAssetUids: this.assetUidMapper, + mappedAssetUrls: this.assetUrlMapper, + }); + } + + entry = lookupEntries( + { + content_type: contentType, + entry, + }, + this.entriesUidMapper, + path.join(this.entriesMapperPath, cTUid, locale), + ); + + const entryResponse = this.stack.contentType(contentType.uid).entry(entry.uid); + Object.assign(entryResponse, entry); + delete entryResponse.publish_details; + apiOptions.apiData = entryResponse; + return apiOptions; + } catch (error) { + console.log('error', error); + } + } + + async enableMandatoryCTReferences(): Promise { + const onSuccess = ({ response: contentType, apiData: { uid } }: any) => { + log(this.importConfig, `${uid} content type references updated`, 'success'); + }; + const onReject = ({ error, apiData: { uid } }: any) => { + log(this.importConfig, formatError(error), 'error'); + throw new Error(`Failed to update references of content type ${uid}`); + }; + return await this.makeConcurrentCall({ + processName: 'Update content type references', + apiContent: this.modifiedCTs, + apiParams: { + serializeData: this.serializeUpdateCTsWithRef.bind(this), + reject: onReject.bind(this), + resolve: onSuccess.bind(this), + entity: 'update-cts', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.importConcurrency, + }); + } - async enableMandatoryCTReferences(): Promise {} + /** + * @method serializeUpdateCTsWithRef + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeUpdateCTsWithRef(apiOptions: ApiOptions): ApiOptions { + const { apiData: contentType } = apiOptions; + if (contentType.field_rules) { + delete contentType.field_rules; + } + lookupExtension( + this.importConfig, + contentType.schema, + this.importConfig.preserveStackVersion, + this.installedExtensions, + ); + const contentTypePayload = this.stack.contentType(contentType.uid); + Object.assign(contentTypePayload, cloneDeep(contentType)); + apiOptions.apiData = contentTypePayload; + return apiOptions; + } async removeAutoCreatedEntries(): Promise {} - async updateFieldRules(): Promise {} + async updateFieldRules(): Promise { + let cTsWithFieldRules = fsUtil.readFile(path.join(this.cTsPath + '/field_rules_uid.json')) as Record[]; + if (!cTsWithFieldRules || cTsWithFieldRules?.length === 0) { + return; + } + for (let cTUid of cTsWithFieldRules) { + const contentType = find(this.cTs, { uid: cTUid }); + if (contentType.field_rules) { + let fieldRuleLength = contentType.field_rules.length; + for (let k = 0; k < fieldRuleLength; k++) { + let fieldRuleConditionLength = contentType.field_rules[k].conditions.length; + for (let i = 0; i < fieldRuleConditionLength; i++) { + if (contentType.field_rules[k].conditions[i].operand_field === 'reference') { + let fieldRulesValue = contentType.field_rules[k].conditions[i].value; + let fieldRulesArray = fieldRulesValue.split('.'); + let updatedValue = []; + for (const element of fieldRulesArray) { + let splittedFieldRulesValue = element; + if (this.entriesUidMapper.hasOwnProperty(splittedFieldRulesValue)) { + updatedValue.push(this.entriesUidMapper[splittedFieldRulesValue]); + } else { + updatedValue.push(element); + } + } + contentType.field_rules[k].conditions[i].value = updatedValue.join('.'); + } + } + } + + const contentTypeResponse: any = await this.stack + .contentType(contentType.uid) + .fetch() + .catch((error) => { + log(this.importConfig, `failed to update the field rules of ${cTUid} ${formatError(error)}`, 'error'); + }); + if (!contentTypeResponse) { + continue; + } + contentTypeResponse.field_rules = contentType.field_rules; + await contentTypeResponse.update().catch((error) => { + log(this.importConfig, `failed to update the field rules of ${cTUid} ${formatError(error)}`, 'error'); + }); + } else { + log(this.importConfig, `No field rules found in content type ${cTUid} to update`, 'error'); + } + } + } + + async publishEntries({ cTUid, locale }: { cTUid: string; locale: string }): Promise { + const processName = 'Publish Entries'; + const indexFileName = 'index.json'; + const basePath = path.join(this.entriesPath, cTUid, locale); + const fs = new FsUtility({ basePath, indexFileName }); + const indexer = fs.indexFileContent; + const indexerCount = values(indexer).length; + const contentType = find(this.cTs, { uid: cTUid }); + + const onSuccess = ({ response, apiData: { environments }, additionalInfo: { entryUid } }: any) => { + log( + this.importConfig, + `Published entry: '${entryUid}' of content type ${cTUid} and locale ${locale} in ${environments?.join( + ',', + )} environments`, + 'info', + ); + }; + const onReject = ({ error, apiData, additionalInfo: { entryUid } }: any) => { + log( + this.importConfig, + `${entryUid} entry of content type ${cTUid} in locale ${locale} failed to publish`, + 'error', + ); + log(this.importConfig, formatError(error), 'error'); + }; - async publishEntries(): Promise {} + for (const index in indexer) { + const chunk = await fs.readChunkFiles.next().catch((error) => { + log(this.importConfig, formatError(error), 'error'); + }); + + if (chunk) { + let apiContent = values(chunk as Record[]); + await this.makeConcurrentCall({ + apiContent, + processName, + indexerCount, + currentIndexer: +index, + apiParams: { + reject: onReject, + resolve: onSuccess, + entity: 'publish-entries', + includeParamOnCompletion: true, + serializeData: this.serializeEntries.bind(this), + additionalInfo: { contentType, locale, cTUid }, + }, + concurrencyLimit: this.importConcurrency, + }).then(() => { + log(this.importConfig, `Published entries for content type ${cTUid} in locale ${locale}`, 'success'); + }); + } + } + } + + /** + * @method serializeEntries + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializePublishEntries(apiOptions: ApiOptions): ApiOptions { + let { apiData: entry, additionalInfo } = apiOptions; + additionalInfo.entryUid = this.entriesUidMapper[entry.uid]; + const requestObject = { + environments: [], + locales: [], + }; + if (entry.publish_details && entry.publish_details.length > 0) { + forEach(entry.publish_details, (pubObject) => { + if ( + this.envs.hasOwnProperty(pubObject.environment) && + indexOf(requestObject.environments, this.envs[pubObject.environment].name) === -1 + ) { + requestObject.environments.push(this.envs[pubObject.environment].name); + } + if (pubObject.locale && indexOf(requestObject.locales, pubObject.locale) === -1) { + requestObject.locales.push(pubObject.locale); + } + }); + } else { + additionalInfo.skip = true; + } + apiOptions.apiData = requestObject; + return apiOptions; + } } diff --git a/packages/contentstack-import/src/utils/entries-helper.ts b/packages/contentstack-import/src/utils/entries-helper.ts index f869058b3f..9d5bd0ea65 100644 --- a/packages/contentstack-import/src/utils/entries-helper.ts +++ b/packages/contentstack-import/src/utils/entries-helper.ts @@ -8,7 +8,7 @@ import config from '../config'; import * as fileHelper from './file-helper'; // update references in entry object -export const lookupEntries = function (data: any, mappedUids: string[], uidMapperPath: string) { +export const lookupEntries = function (data: any, mappedUids: Record, uidMapperPath: string) { let parent: string[] = []; let uids: string[] = []; let unmapped: string[] = []; @@ -426,3 +426,160 @@ function doEntryReferencesExist(element: Record[] | any): boolean { function isEntryRef(element: any) { return element.type === 'reference' && element.attrs.type === 'entry'; } + +export const restoreJsonRteEntryRefs = ( + entry: Record, + sourceStackEntry, + ctSchema, + { mappedAssetUids, mappedAssetUrls }, +) => { + // let mappedAssetUids = fileHelper.readFileSync(this.mappedAssetUidPath) || {}; + // let mappedAssetUrls = fileHelper.readFileSync(this.mappedAssetUrlPath) || {}; + for (const element of ctSchema) { + switch (element.data_type) { + case 'blocks': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e, eIndex) => { + let key = Object.keys(e).pop(); + let subBlock = element.blocks.filter((block) => block.uid === key).pop(); + let sourceStackElement = sourceStackEntry[element.uid][eIndex][key]; + e[key] = restoreJsonRteEntryRefs(e[key], sourceStackElement, subBlock.schema, { + mappedAssetUids, + mappedAssetUrls, + }); + return e; + }); + } + } + break; + } + case 'global_field': + case 'group': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e, eIndex) => { + let sourceStackElement = sourceStackEntry[element.uid][eIndex]; + e = restoreJsonRteEntryRefs(e, sourceStackElement, element.schema, { mappedAssetUids, mappedAssetUrls }); + return e; + }); + } else { + let sourceStackElement = sourceStackEntry[element.uid]; + entry[element.uid] = restoreJsonRteEntryRefs(entry[element.uid], sourceStackElement, element.schema, { + mappedAssetUids, + mappedAssetUrls, + }); + } + } + break; + } + case 'json': { + if (entry[element.uid] && element.field_metadata.rich_text_type) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((field, index) => { + // i am facing a Maximum call stack exceeded issue, + // probably because of this loop operation + + let entryRefs = sourceStackEntry[element.uid][index].children + .map((e, i) => { + return { index: i, value: e }; + }) + .filter((e) => doEntryReferencesExist(e.value)) + .map((e) => { + // commenting the line below resolved the maximum call stack exceeded issue + // e.value = this.setDirtyTrue(e.value) + setDirtyTrue(e.value); + return e; + }) + .map((e) => { + // commenting the line below resolved the maximum call stack exceeded issue + // e.value = this.resolveAssetRefsInEntryRefsForJsonRte(e, mappedAssetUids, mappedAssetUrls) + resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls); + return e; + }); + + if (entryRefs.length > 0) { + entryRefs.forEach((entryRef) => { + field.children.splice(entryRef.index, 0, entryRef.value); + }); + } + return field; + }); + } else { + let entryRefs = sourceStackEntry[element.uid].children + .map((e, index) => { + return { index: index, value: e }; + }) + .filter((e) => doEntryReferencesExist(e.value)) + .map((e) => { + setDirtyTrue(e.value); + return e; + }) + .map((e) => { + resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls); + return e; + }); + + if (entryRefs.length > 0) { + entryRefs.forEach((entryRef) => { + if (!_.isEmpty(entry[element.uid]) && entry[element.uid].children) { + entry[element.uid].children.splice(entryRef.index, 0, entryRef.value); + } + }); + } + } + } + break; + } + } + } + return entry; +}; + +function setDirtyTrue(jsonRteChild: Record) { + // also removing uids in this function + if (jsonRteChild.type) { + if (_.isObject(jsonRteChild.attrs)) { + jsonRteChild.attrs['dirty'] = true; + } + delete jsonRteChild.uid; + + if (jsonRteChild.children && jsonRteChild.children.length > 0) { + jsonRteChild.children = jsonRteChild.children.map((subElement) => this.setDirtyTrue(subElement)); + } + } + return jsonRteChild; +} + +function resolveAssetRefsInEntryRefsForJsonRte(jsonRteChild, mappedAssetUids, mappedAssetUrls) { + if (jsonRteChild.type) { + if (jsonRteChild.attrs.type === 'asset') { + let assetUrl; + if (mappedAssetUids[jsonRteChild.attrs['asset-uid']]) { + jsonRteChild.attrs['asset-uid'] = mappedAssetUids[jsonRteChild.attrs['asset-uid']]; + } + + if (jsonRteChild.attrs['display-type'] !== 'link') { + assetUrl = jsonRteChild.attrs['asset-link']; + } else { + assetUrl = jsonRteChild.attrs['href']; + } + + if (mappedAssetUrls[assetUrl]) { + if (jsonRteChild.attrs['display-type'] !== 'link') { + jsonRteChild.attrs['asset-link'] = mappedAssetUrls[assetUrl]; + } else { + jsonRteChild.attrs['href'] = mappedAssetUrls[assetUrl]; + } + } + } + + if (jsonRteChild.children && jsonRteChild.children.length > 0) { + jsonRteChild.children = jsonRteChild.children.map((subElement) => + resolveAssetRefsInEntryRefsForJsonRte(subElement, mappedAssetUids, mappedAssetUrls), + ); + } + } + + return jsonRteChild; +} diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index f794713082..19cfca611e 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -20,5 +20,10 @@ export { } from './marketplace-app-helper'; export { schemaTemplate, suppressSchemaReference, removeReferenceFields } from './content-type-helper'; export { lookupExtension } from './extension-helper'; -export { lookupEntries, removeUidsFromJsonRteFields, removeEntryRefsFromJSONRTE } from './entries-helper'; +export { + lookupEntries, + removeUidsFromJsonRteFields, + removeEntryRefsFromJSONRTE, + restoreJsonRteEntryRefs, +} from './entries-helper'; export * from './common-helper'; From 129d51d2b5cfd2d5b3c280f74937f20357f961e4 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Thu, 3 Aug 2023 18:54:20 +0530 Subject: [PATCH 3/5] fixed issues --- .../src/import/modules/base-class.ts | 2 +- .../src/import/modules/entries.ts | 34 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 167d678148..4fbe6f04f0 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -351,7 +351,7 @@ export default abstract class BaseClass { .then(onSuccess) .catch(onReject); case 'update-entries': - return apiData.update().then(onSuccess).catch(onReject); + return apiData.update({ locale: additionalInfo.locale }).then(onSuccess).catch(onReject); case 'publish-entries': if (additionalInfo.skip) { return Promise.resolve(onSuccess(apiData)); diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index c87e1d8f96..4f001524af 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -152,11 +152,6 @@ export default class EntriesImport extends BaseClass { log(this.importConfig, 'Publishing entries', 'info'); this.envs = fileHelper.readFileSync(this.envPath); for (let entryRequestOption of entryRequestOptions) { - log( - this.importConfig, - `Starting publish entries for ${entryRequestOption.cTUid} in locale ${entryRequestOption.locale}`, - 'info', - ); await this.publishEntries(entryRequestOption).catch((error) => { log( this.importConfig, @@ -285,12 +280,12 @@ export default class EntriesImport extends BaseClass { }); const contentType = find(this.cTs, { uid: cTUid }); - const onSuccess = ({ response, apiData: { uid, url, title }, additionalInfo: { entryFileName } }: any) => { - log(this.importConfig, `Created entry: '${title}' of content type ${cTUid} in locale ${locale}`, 'info'); - this.entriesUidMapper[uid] = response.uid; - response.sourceEntryFilePath = path.join(basePath, entryFileName); // stores source file path temporarily - response.entryOldUid = uid; // stores old uid temporarily - entriesCreateFileHelper.writeIntoFile({ [response.uid]: response } as any, { mapKeyVal: true }); + const onSuccess = ({ response, apiData: entry, additionalInfo: { entryFileName } }: any) => { + log(this.importConfig, `Created entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info'); + this.entriesUidMapper[entry.uid] = response.uid; + entry.sourceEntryFilePath = path.join(basePath, entryFileName); // stores source file path temporarily + entry.entryOldUid = entry.uid; // stores old uid temporarily + entriesCreateFileHelper.writeIntoFile({ [response.uid]: entry } as any, { mapKeyVal: true }); }; const onReject = ({ error, apiData: { uid, title } }: any) => { log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to create`, 'error'); @@ -353,7 +348,7 @@ export default class EntriesImport extends BaseClass { }, this.assetUidMapper, this.assetUrlMapper, - path.join(this.entriesPath, cTUid, locale), + path.join(this.entriesPath, cTUid), this.installedExtensions, ); delete entry.publish_details; @@ -394,7 +389,7 @@ export default class EntriesImport extends BaseClass { const onReject = ({ error, apiData: { uid, title } }: any) => { log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to update`, 'error'); log(this.importConfig, formatError(error), 'error'); - this.failedEntries.push({ content_type: cTUid, locale, entry: { uid, title } }); + this.failedEntries.push({ content_type: cTUid, locale, entry: { uid: this.entriesUidMapper[uid], title } }); }; for (const index in indexer) { @@ -459,8 +454,8 @@ export default class EntriesImport extends BaseClass { path.join(this.entriesMapperPath, cTUid, locale), ); - const entryResponse = this.stack.contentType(contentType.uid).entry(entry.uid); - Object.assign(entryResponse, entry); + const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry.uid]); + Object.assign(entryResponse, cloneDeep(entry)); delete entryResponse.publish_details; apiOptions.apiData = entryResponse; return apiOptions; @@ -543,7 +538,6 @@ export default class EntriesImport extends BaseClass { } } } - const contentTypeResponse: any = await this.stack .contentType(contentType.uid) .fetch() @@ -557,6 +551,7 @@ export default class EntriesImport extends BaseClass { await contentTypeResponse.update().catch((error) => { log(this.importConfig, `failed to update the field rules of ${cTUid} ${formatError(error)}`, 'error'); }); + log(this.importConfig, `Updated the field rules of ${cTUid}`, 'info'); } else { log(this.importConfig, `No field rules found in content type ${cTUid} to update`, 'error'); } @@ -572,6 +567,11 @@ export default class EntriesImport extends BaseClass { const indexerCount = values(indexer).length; const contentType = find(this.cTs, { uid: cTUid }); + if (indexerCount === 0) { + return Promise.resolve(); + } + log(this.importConfig, `Starting publish entries for ${cTUid} in locale ${locale}`, 'info'); + const onSuccess = ({ response, apiData: { environments }, additionalInfo: { entryUid } }: any) => { log( this.importConfig, @@ -607,7 +607,7 @@ export default class EntriesImport extends BaseClass { resolve: onSuccess, entity: 'publish-entries', includeParamOnCompletion: true, - serializeData: this.serializeEntries.bind(this), + serializeData: this.serializePublishEntries.bind(this), additionalInfo: { contentType, locale, cTUid }, }, concurrencyLimit: this.importConcurrency, From 9aaef4aa977c0889d6664e1b770c5938eb176627 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Fri, 4 Aug 2023 12:24:34 +0530 Subject: [PATCH 4/5] added removal of autocreated entries in masterlocale feature --- packages/contentstack-export/tsconfig.json | 2 +- .../src/import/modules/base-class.ts | 12 +++- .../src/import/modules/entries.ts | 56 +++++++++++++------ packages/contentstack-import/tsconfig.json | 2 +- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/packages/contentstack-export/tsconfig.json b/packages/contentstack-export/tsconfig.json index da8fc4b724..5136623392 100644 --- a/packages/contentstack-export/tsconfig.json +++ b/packages/contentstack-export/tsconfig.json @@ -12,7 +12,7 @@ "skipLibCheck": true, "sourceMap": false, "esModuleInterop": true, - "noImplicitAny": false, + "noImplicitAny": true, "lib": [ "ES2019", "es2020.promise" diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 4fbe6f04f0..f2efc86e3c 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -46,7 +46,8 @@ export type ApiModuleType = | 'create-custom-role' | 'create-entries' | 'update-entries' - | 'publish-entries'; + | 'publish-entries' + | 'delete-entries'; export type ApiOptions = { uid?: string; @@ -347,7 +348,7 @@ export default abstract class BaseClass { return this.stack .contentType(additionalInfo.cTUid) .entry() - .create({ entry: apiData }) + .create({ entry: apiData }, { locale: additionalInfo.locale }) .then(onSuccess) .catch(onReject); case 'update-entries': @@ -362,6 +363,13 @@ export default abstract class BaseClass { .publish({ publishDetails: apiData, locale: additionalInfo.locale }) .then(onSuccess) .catch(onReject); + case 'delete-entries': + return this.stack + .contentType(apiData.cTUid) + .entry(apiData.entryUid) + .delete({ locale: this.importConfig?.master_locale?.code }) + .then(onSuccess) + .catch(onReject); default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 4f001524af..129698a16c 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -52,6 +52,7 @@ export default class EntriesImport extends BaseClass { private locales: Record[]; private entriesUidMapper: Record; private envs: Record; + private autoCreatedEntries: Record[]; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -78,19 +79,11 @@ export default class EntriesImport extends BaseClass { this.jsonRteCTs = []; this.jsonRteCTsWithRef = []; this.envs = {}; + this.autoCreatedEntries = []; } async start(): Promise { try { - /** - * Read CTS - * Suppress CTS - * Read locales - * Create request objs - * create entries - * update entries - */ - this.cTs = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record[]; if (!this.cTs || isEmpty(this.cTs)) { log(this.importConfig, 'No content type found', 'info'); @@ -132,14 +125,19 @@ export default class EntriesImport extends BaseClass { log(this.importConfig, 'Restoring content type changes', 'info'); await this.enableMandatoryCTReferences().catch((error) => { - log(this.importConfig, `failed update content type references ${formatError(error)}`, 'error'); - }); - - log(this.importConfig, 'Removing entries from master language which got created by default', 'info'); - await this.removeAutoCreatedEntries().catch((error) => { - log(this.importConfig, `failed to remove auto created entries in master locale ${formatError(error)}`, 'error'); + log(this.importConfig, `Error while updating content type references ${formatError(error)}`, 'error'); }); + if (this.autoCreatedEntries.length > 0) { + log(this.importConfig, 'Removing entries from master language which got created by default', 'info'); + await this.removeAutoCreatedEntries().catch((error) => { + log( + this.importConfig, + `Error while removing auto created entries in master locale ${formatError(error)}`, + 'error', + ); + }); + } // Update field rule of content types which are got removed earlier log(this.importConfig, 'Updating the field rules of content type', 'info'); await this.updateFieldRules().catch((error) => { @@ -269,6 +267,7 @@ export default class EntriesImport extends BaseClass { return Promise.resolve(); } log(this.importConfig, `Starting to create entries for ${cTUid} in locale ${locale}`, 'info'); + const isMasterLocale = locale === this.importConfig?.master_locale?.code; // Write created entries const entriesCreateFileHelper = new FsUtility({ moduleName: 'created-entries', @@ -285,6 +284,9 @@ export default class EntriesImport extends BaseClass { this.entriesUidMapper[entry.uid] = response.uid; entry.sourceEntryFilePath = path.join(basePath, entryFileName); // stores source file path temporarily entry.entryOldUid = entry.uid; // stores old uid temporarily + if (!isMasterLocale) { + this.autoCreatedEntries.push({ cTUid, locale, entryUid: response.uid }); + } entriesCreateFileHelper.writeIntoFile({ [response.uid]: entry } as any, { mapKeyVal: true }); }; const onReject = ({ error, apiData: { uid, title } }: any) => { @@ -508,7 +510,29 @@ export default class EntriesImport extends BaseClass { return apiOptions; } - async removeAutoCreatedEntries(): Promise {} + async removeAutoCreatedEntries(): Promise { + const onSuccess = ({ response, apiData: { entryUid } }: any) => { + log(this.importConfig, `Auto created entry in master locale removed - entry uid ${entryUid} `, 'success'); + }; + const onReject = ({ error, apiData: { entryUid } }: any) => { + log( + this.importConfig, + `Failed to remove auto created entry in master locale - entry uid ${entryUid} \n ${formatError(error)}`, + 'error', + ); + }; + return await this.makeConcurrentCall({ + processName: 'Remove auto created entry in master locale', + apiContent: this.autoCreatedEntries, + apiParams: { + reject: onReject.bind(this), + resolve: onSuccess.bind(this), + entity: 'delete-entries', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.importConcurrency, + }); + } async updateFieldRules(): Promise { let cTsWithFieldRules = fsUtil.readFile(path.join(this.cTsPath + '/field_rules_uid.json')) as Record[]; diff --git a/packages/contentstack-import/tsconfig.json b/packages/contentstack-import/tsconfig.json index 44beaaf137..679afbeaf5 100644 --- a/packages/contentstack-import/tsconfig.json +++ b/packages/contentstack-import/tsconfig.json @@ -11,7 +11,7 @@ "skipLibCheck": true, "sourceMap": false, "esModuleInterop": true, - "noImplicitAny": false, + "noImplicitAny": true, "lib": [ "es2019", "es2020.promise", From f9cbeead8c190459a7434633f3f069fb6aad51a4 Mon Sep 17 00:00:00 2001 From: Shafeeq PP Date: Wed, 9 Aug 2023 12:15:43 +0530 Subject: [PATCH 5/5] fixed type issues --- .../src/import/modules/entries.ts | 9 ++-- .../src/utils/entries-helper.ts | 46 +++++++++---------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index 129698a16c..2d711d9afa 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -435,7 +435,7 @@ export default class EntriesImport extends BaseClass { try { const sourceEntryFilePath = entry.sourceEntryFilePath; - const sourceEntry = (fsUtil.readFile(sourceEntryFilePath) || {})[entry.entryOldUid]; + const sourceEntry = ((fsUtil.readFile(sourceEntryFilePath) || {}) as Record)[entry.entryOldUid]; // Removing temp values delete entry.sourceEntryFilePath; delete entry.entryOldUid; @@ -572,7 +572,7 @@ export default class EntriesImport extends BaseClass { continue; } contentTypeResponse.field_rules = contentType.field_rules; - await contentTypeResponse.update().catch((error) => { + await contentTypeResponse.update().catch((error: Error) => { log(this.importConfig, `failed to update the field rules of ${cTUid} ${formatError(error)}`, 'error'); }); log(this.importConfig, `Updated the field rules of ${cTUid}`, 'info'); @@ -650,7 +650,10 @@ export default class EntriesImport extends BaseClass { serializePublishEntries(apiOptions: ApiOptions): ApiOptions { let { apiData: entry, additionalInfo } = apiOptions; additionalInfo.entryUid = this.entriesUidMapper[entry.uid]; - const requestObject = { + const requestObject: { + environments: Array; + locales: Array; + } = { environments: [], locales: [], }; diff --git a/packages/contentstack-import/src/utils/entries-helper.ts b/packages/contentstack-import/src/utils/entries-helper.ts index 9d5bd0ea65..c0d56d42ce 100644 --- a/packages/contentstack-import/src/utils/entries-helper.ts +++ b/packages/contentstack-import/src/utils/entries-helper.ts @@ -268,7 +268,7 @@ export const removeUidsFromJsonRteFields = ( case 'group': { if (entry[element.uid]) { if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e) => { + entry[element.uid] = entry[element.uid].map((e: any) => { e = removeUidsFromJsonRteFields(e, element.schema); return e; }); @@ -362,7 +362,7 @@ export const removeEntryRefsFromJSONRTE = (entry: Record, ctSchema: case 'group': { if (entry[element.uid]) { if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e) => { + entry[element.uid] = entry[element.uid].map((e: any) => { e = removeEntryRefsFromJSONRTE(e, element.schema); return e; }); @@ -429,9 +429,9 @@ function isEntryRef(element: any) { export const restoreJsonRteEntryRefs = ( entry: Record, - sourceStackEntry, - ctSchema, - { mappedAssetUids, mappedAssetUrls }, + sourceStackEntry: any, + ctSchema: any, + { mappedAssetUids, mappedAssetUrls }: any, ) => { // let mappedAssetUids = fileHelper.readFileSync(this.mappedAssetUidPath) || {}; // let mappedAssetUrls = fileHelper.readFileSync(this.mappedAssetUrlPath) || {}; @@ -440,9 +440,9 @@ export const restoreJsonRteEntryRefs = ( case 'blocks': { if (entry[element.uid]) { if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e, eIndex) => { + entry[element.uid] = entry[element.uid].map((e: any, eIndex: number) => { let key = Object.keys(e).pop(); - let subBlock = element.blocks.filter((block) => block.uid === key).pop(); + let subBlock = element.blocks.filter((block: any) => block.uid === key).pop(); let sourceStackElement = sourceStackEntry[element.uid][eIndex][key]; e[key] = restoreJsonRteEntryRefs(e[key], sourceStackElement, subBlock.schema, { mappedAssetUids, @@ -458,7 +458,7 @@ export const restoreJsonRteEntryRefs = ( case 'group': { if (entry[element.uid]) { if (element.multiple) { - entry[element.uid] = entry[element.uid].map((e, eIndex) => { + entry[element.uid] = entry[element.uid].map((e: any, eIndex: number) => { let sourceStackElement = sourceStackEntry[element.uid][eIndex]; e = restoreJsonRteEntryRefs(e, sourceStackElement, element.schema, { mappedAssetUids, mappedAssetUrls }); return e; @@ -476,22 +476,22 @@ export const restoreJsonRteEntryRefs = ( case 'json': { if (entry[element.uid] && element.field_metadata.rich_text_type) { if (element.multiple) { - entry[element.uid] = entry[element.uid].map((field, index) => { + entry[element.uid] = entry[element.uid].map((field: any, index: number) => { // i am facing a Maximum call stack exceeded issue, // probably because of this loop operation let entryRefs = sourceStackEntry[element.uid][index].children - .map((e, i) => { + .map((e: any, i: number) => { return { index: i, value: e }; }) - .filter((e) => doEntryReferencesExist(e.value)) - .map((e) => { + .filter((e: any) => doEntryReferencesExist(e.value)) + .map((e: any) => { // commenting the line below resolved the maximum call stack exceeded issue // e.value = this.setDirtyTrue(e.value) setDirtyTrue(e.value); return e; }) - .map((e) => { + .map((e: any) => { // commenting the line below resolved the maximum call stack exceeded issue // e.value = this.resolveAssetRefsInEntryRefsForJsonRte(e, mappedAssetUids, mappedAssetUrls) resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls); @@ -499,7 +499,7 @@ export const restoreJsonRteEntryRefs = ( }); if (entryRefs.length > 0) { - entryRefs.forEach((entryRef) => { + entryRefs.forEach((entryRef: any) => { field.children.splice(entryRef.index, 0, entryRef.value); }); } @@ -507,21 +507,21 @@ export const restoreJsonRteEntryRefs = ( }); } else { let entryRefs = sourceStackEntry[element.uid].children - .map((e, index) => { + .map((e: any, index: number) => { return { index: index, value: e }; }) - .filter((e) => doEntryReferencesExist(e.value)) - .map((e) => { + .filter((e: any) => doEntryReferencesExist(e.value)) + .map((e: any) => { setDirtyTrue(e.value); return e; }) - .map((e) => { + .map((e: any) => { resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls); return e; }); if (entryRefs.length > 0) { - entryRefs.forEach((entryRef) => { + entryRefs.forEach((entryRef: any) => { if (!_.isEmpty(entry[element.uid]) && entry[element.uid].children) { entry[element.uid].children.splice(entryRef.index, 0, entryRef.value); } @@ -536,7 +536,7 @@ export const restoreJsonRteEntryRefs = ( return entry; }; -function setDirtyTrue(jsonRteChild: Record) { +function setDirtyTrue(jsonRteChild: any) { // also removing uids in this function if (jsonRteChild.type) { if (_.isObject(jsonRteChild.attrs)) { @@ -545,13 +545,13 @@ function setDirtyTrue(jsonRteChild: Record) { delete jsonRteChild.uid; if (jsonRteChild.children && jsonRteChild.children.length > 0) { - jsonRteChild.children = jsonRteChild.children.map((subElement) => this.setDirtyTrue(subElement)); + jsonRteChild.children = jsonRteChild.children.map((subElement: any) => this.setDirtyTrue(subElement)); } } return jsonRteChild; } -function resolveAssetRefsInEntryRefsForJsonRte(jsonRteChild, mappedAssetUids, mappedAssetUrls) { +function resolveAssetRefsInEntryRefsForJsonRte(jsonRteChild: any, mappedAssetUids: any, mappedAssetUrls: any) { if (jsonRteChild.type) { if (jsonRteChild.attrs.type === 'asset') { let assetUrl; @@ -575,7 +575,7 @@ function resolveAssetRefsInEntryRefsForJsonRte(jsonRteChild, mappedAssetUids, ma } if (jsonRteChild.children && jsonRteChild.children.length > 0) { - jsonRteChild.children = jsonRteChild.children.map((subElement) => + jsonRteChild.children = jsonRteChild.children.map((subElement: any) => resolveAssetRefsInEntryRefsForJsonRte(subElement, mappedAssetUids, mappedAssetUrls), ); }