diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index f423abef4f..92edaa0001 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -137,7 +137,7 @@ const config: DefaultConfig = { // total no of entries fetched in each content type in a single call limit: 100, dependencies: ['locales', 'content-types'], - exportVersions: true, + exportVersions: false, }, extensions: { dirName: 'extensions', diff --git a/packages/contentstack-export/src/export/modules/entries.ts b/packages/contentstack-export/src/export/modules/entries.ts index 15557cbbdd..f795cee78c 100644 --- a/packages/contentstack-export/src/export/modules/entries.ts +++ b/packages/contentstack-export/src/export/modules/entries.ts @@ -56,7 +56,7 @@ export default class EntriesExport extends BaseClass { for (let entryRequestOption of entryRequestOptions) { log( this.exportConfig, - `Starting export of entries of contenttype - ${entryRequestOption.contentType} locale - ${entryRequestOption.locale}`, + `Starting export of entries of content type - ${entryRequestOption.contentType} locale - ${entryRequestOption.locale}`, 'info', ); await this.getEntries(entryRequestOption); diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index f9d6c1097e..849e6c215c 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -7,7 +7,7 @@ "dependencies": { "@contentstack/cli-command": "^1.2.11", "@contentstack/cli-utilities": "^1.5.1", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.10.0", "@oclif/config": "^1.18.3", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", @@ -96,4 +96,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} +} \ No newline at end of file diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 20b1f4626e..4a5621fd05 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -377,7 +377,8 @@ const config: DefaultConfig = { 'content-types', 'webhooks', 'custom-roles', - 'workflows' + 'workflows', + 'entries', ], rateLimit: 5, preserveStackVersion: false, @@ -389,7 +390,7 @@ const config: DefaultConfig = { developerHubBaseUrl: '', marketplaceAppEncryptionKey: 'nF2ejRQcTv', getEncryptionKeyMaxRetry: 3, - useNewModuleStructure: true + useNewModuleStructure: true, // useBackedupDir: '', // backupConcurrency: 10, }; diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index d7e303f235..f2efc86e3c 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -43,7 +43,11 @@ export type ApiModuleType = | 'update-labels' | 'create-webhooks' | 'create-workflows' - | 'create-custom-role'; + | 'create-custom-role' + | 'create-entries' + | 'update-entries' + | 'publish-entries' + | 'delete-entries'; export type ApiOptions = { uid?: string; @@ -231,7 +235,7 @@ export default abstract class BaseClass { apiOptions = apiOptions.serializeData(apiOptions); } - const { uid, entity, reject, resolve, apiData, additionalInfo, includeParamOnCompletion } = apiOptions; + const { uid, entity, reject, resolve, apiData, additionalInfo = {}, includeParamOnCompletion } = apiOptions; const onSuccess = (response: any) => resolve({ @@ -295,6 +299,9 @@ export default abstract class BaseClass { case 'create-cts': return this.stack.contentType().create(apiData).then(onSuccess).catch(onReject); case 'update-cts': + if (additionalInfo.skip) { + return Promise.resolve(onSuccess(apiData)); + } return apiData.update().then(onSuccess).catch(onReject); case 'update-gfs': return apiData.update().then(onSuccess).catch(onReject); @@ -337,7 +344,32 @@ export default abstract class BaseClass { .create({ role: apiData as RoleData }) .then(onSuccess) .catch(onReject); - + case 'create-entries': + return this.stack + .contentType(additionalInfo.cTUid) + .entry() + .create({ entry: apiData }, { locale: additionalInfo.locale }) + .then(onSuccess) + .catch(onReject); + case 'update-entries': + return apiData.update({ locale: additionalInfo.locale }).then(onSuccess).catch(onReject); + case 'publish-entries': + if (additionalInfo.skip) { + return Promise.resolve(onSuccess(apiData)); + } + return this.stack + .contentType(additionalInfo.cTUid) + .entry(additionalInfo.entryUid) + .publish({ publishDetails: apiData, locale: additionalInfo.locale }) + .then(onSuccess) + .catch(onReject); + case 'delete-entries': + return this.stack + .contentType(apiData.cTUid) + .entry(apiData.entryUid) + .delete({ locale: this.importConfig?.master_locale?.code }) + .then(onSuccess) + .catch(onReject); default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts new file mode 100644 index 0000000000..2d711d9afa --- /dev/null +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -0,0 +1,678 @@ +/* eslint-disable no-prototype-builtins */ +/*! + * Contentstack Import + * Copyright (c) 2019 Contentstack LLC + * MIT Licensed + */ + +import * as path from 'path'; +import { isEmpty, values, cloneDeep, find, indexOf, forEach } from 'lodash'; +import { ContentType, FsUtility } from '@contentstack/cli-utilities'; +import { + fsUtil, + log, + formatError, + lookupExtension, + suppressSchemaReference, + removeUidsFromJsonRteFields, + removeEntryRefsFromJSONRTE, + restoreJsonRteEntryRefs, + lookupEntries, + lookupAssets, + fileHelper, +} from '../../utils'; +import { ModuleClassParams } from '../../types'; +import BaseClass, { ApiOptions } from './base-class'; + +export default class EntriesImport extends BaseClass { + private assetUidMapperPath: string; + private assetUidMapper: Record; + private assetUrlMapper: Record; + private assetUrlMapperPath: string; + private entriesMapperPath: string; + private envPath: string; + private entriesUIDMapperPath: string; + private uniqueUidMapperPath: string; + private modifiedCTsPath: string; + private marketplaceAppMapperPath: string; + private entriesConfig: Record; + private cTsPath: string; + private localesPath: string; + private importConcurrency: number; + private entriesPath: string; + private cTs: Record[]; + private modifiedCTs: Record[]; + private refCTs: string[]; + private jsonRteCTs: Record; + private jsonRteCTsWithRef: Record; + private jsonRteEntries: Record; + private installedExtensions: Record[]; + private createdEntries: Record[]; + private failedEntries: Record[]; + private locales: Record[]; + private entriesUidMapper: Record; + private envs: Record; + private autoCreatedEntries: Record[]; + + constructor({ importConfig, stackAPIClient }: ModuleClassParams) { + super({ importConfig, stackAPIClient }); + this.assetUidMapperPath = path.resolve(importConfig.data, 'mapper', 'assets', 'uid-mapping.json'); + this.assetUrlMapperPath = path.resolve(importConfig.data, 'mapper', 'assets', 'url-mapping.json'); + this.entriesMapperPath = path.resolve(importConfig.data, 'mapper', 'entries'); + this.envPath = path.resolve(importConfig.data, 'environments', 'environments.json'); + this.entriesUIDMapperPath = path.join(this.entriesMapperPath, 'uid-mapping.json'); + this.uniqueUidMapperPath = path.join(this.entriesMapperPath, 'unique-mapping.json'); + this.modifiedCTsPath = path.join(this.entriesMapperPath, 'modified-schemas.json'); + this.marketplaceAppMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json'); + this.entriesConfig = importConfig.modules.entries; + this.entriesPath = path.resolve(importConfig.data, this.entriesConfig.dirName); + this.cTsPath = path.resolve(importConfig.data, importConfig.modules['content-types'].dirName); + this.localesPath = path.resolve( + importConfig.data, + importConfig.modules.locales.dirName, + importConfig.modules.locales.fileName, + ); + this.importConcurrency = this.entriesConfig.importConcurrency || importConfig.importConcurrency; + this.entriesUidMapper = {}; + this.modifiedCTs = []; + this.refCTs = []; + this.jsonRteCTs = []; + this.jsonRteCTsWithRef = []; + this.envs = {}; + this.autoCreatedEntries = []; + } + + async start(): Promise { + try { + this.cTs = fsUtil.readFile(path.join(this.cTsPath, 'schema.json')) as Record[]; + if (!this.cTs || isEmpty(this.cTs)) { + log(this.importConfig, 'No content type found', 'info'); + return; + } + this.installedExtensions = ( + ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} } + ).extension_uid; + + this.assetUidMapper = (fsUtil.readFile(this.assetUidMapperPath) as Record) || {}; + this.assetUrlMapper = (fsUtil.readFile(this.assetUrlMapperPath) as Record) || {}; + + fsUtil.makeDirectory(this.entriesMapperPath); + await this.disableMandatoryCTReferences(); + this.locales = values(fsUtil.readFile(this.localesPath) as Record[]); + this.locales.unshift(this.importConfig.master_locale); // adds master locale to the list + + //Create Entries + const entryRequestOptions = this.populateEntryCreatePayload(); + for (let entryRequestOption of entryRequestOptions) { + await this.createEntries(entryRequestOption); + } + await fileHelper.writeLargeFile(path.join(this.entriesMapperPath, 'uid-mapping.json'), this.entriesUidMapper); // TBD: manages mapper in one file, should find an alternative + fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); + + // Update entries with references + const entryUpdateRequestOptions = this.populateEntryUpdatePayload(); + for (let entryUpdateRequestOption of entryUpdateRequestOptions) { + await this.updateEntriesWithReferences(entryUpdateRequestOption).catch((error) => { + log( + this.importConfig, + `Error while updating entries references of ${entryUpdateRequestOption.cTUid} in locale ${entryUpdateRequestOption.locale}`, + 'error', + ); + log(this.importConfig, formatError(error), 'error'); + }); + } + fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); + + log(this.importConfig, 'Restoring content type changes', 'info'); + await this.enableMandatoryCTReferences().catch((error) => { + log(this.importConfig, `Error while updating content type references ${formatError(error)}`, 'error'); + }); + + if (this.autoCreatedEntries.length > 0) { + log(this.importConfig, 'Removing entries from master language which got created by default', 'info'); + await this.removeAutoCreatedEntries().catch((error) => { + log( + this.importConfig, + `Error while removing auto created entries in master locale ${formatError(error)}`, + 'error', + ); + }); + } + // Update field rule of content types which are got removed earlier + log(this.importConfig, 'Updating the field rules of content type', 'info'); + await this.updateFieldRules().catch((error) => { + log(this.importConfig, `Error while updating field rules of content type ${formatError(error)}`, 'error'); + }); + log(this.importConfig, 'Entries imported successfully', 'success'); + + // Publishing entries + if (this.importConfig.entriesPublish) { + log(this.importConfig, 'Publishing entries', 'info'); + this.envs = fileHelper.readFileSync(this.envPath); + for (let entryRequestOption of entryRequestOptions) { + await this.publishEntries(entryRequestOption).catch((error) => { + log( + this.importConfig, + `Error in publishing entries of ${entryRequestOption.cTUid} in locale ${ + entryRequestOption.locale + } ${formatError(error)}`, + 'error', + ); + }); + } + log(this.importConfig, 'All the entries have been published successfully', 'success'); + } + } catch (error) { + log(this.importConfig, formatError(error), 'error'); + throw new Error('Error while importing entries'); + } + } + + async disableMandatoryCTReferences() { + const onSuccess = ({ response: contentType, apiData: { uid } }: any) => { + log(this.importConfig, `${uid} content type references removed temporarily`, 'success'); + }; + const onReject = ({ error, apiData: { uid } }: any) => { + log(this.importConfig, formatError(error), 'error'); + throw new Error(`${uid} content type references removal failed`); + }; + return await this.makeConcurrentCall({ + processName: 'Update content types (removing mandatory references temporarily)', + apiContent: this.cTs, + apiParams: { + serializeData: this.serializeUpdateCTs.bind(this), + reject: onReject.bind(this), + resolve: onSuccess.bind(this), + entity: 'update-cts', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.importConcurrency, + }).then(() => { + fsUtil.writeFile(this.modifiedCTsPath, this.modifiedCTs); + }); + } + + /** + * @method serializeUpdateCTs + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeUpdateCTs(apiOptions: ApiOptions): ApiOptions { + const { apiData: contentType } = apiOptions; + if (contentType.field_rules) { + delete contentType.field_rules; + } + const flag = { + suppressed: false, + references: false, + jsonRte: false, + jsonRteEmbeddedEntries: false, + }; + suppressSchemaReference(contentType.schema, flag); + // Check if suppress modified flag + if (flag.suppressed) { + this.modifiedCTs.push(find(this.cTs, { uid: contentType.uid })); + } else { + // Note: Skips the content type from update if no reference found + apiOptions.additionalInfo = { skip: true }; + return apiOptions; + } + + if (flag.references) { + this.refCTs.push(contentType.uid); + } + + if (flag.jsonRte) { + this.jsonRteCTs.push(contentType.uid); + if (flag.jsonRteEmbeddedEntries) { + this.jsonRteCTsWithRef.push(contentType.uid); + if (this.refCTs.indexOf(contentType.uid) === -1) { + this.refCTs.push(contentType.uid); + } + } + } + lookupExtension( + this.importConfig, + contentType.schema, + this.importConfig.preserveStackVersion, + this.installedExtensions, + ); + const contentTypePayload = this.stack.contentType(contentType.uid); + Object.assign(contentTypePayload, cloneDeep(contentType)); + apiOptions.apiData = contentTypePayload; + return apiOptions; + } + + populateEntryCreatePayload(): { cTUid: string; locale: string }[] { + const requestOptions: { cTUid: string; locale: string }[] = []; + for (let locale of this.locales) { + for (let contentType of this.cTs) { + requestOptions.push({ + cTUid: contentType.uid, + locale: locale.code, + }); + } + } + return requestOptions; + } + + async createEntries({ cTUid, locale }: { cTUid: string; locale: string }): Promise { + const processName = 'Create Entries'; + const indexFileName = 'index.json'; + const basePath = path.join(this.entriesPath, cTUid, locale); + const fs = new FsUtility({ basePath, indexFileName }); + const indexer = fs.indexFileContent; + const indexerCount = values(indexer).length; + if (indexerCount === 0) { + return Promise.resolve(); + } + log(this.importConfig, `Starting to create entries for ${cTUid} in locale ${locale}`, 'info'); + const isMasterLocale = locale === this.importConfig?.master_locale?.code; + // Write created entries + const entriesCreateFileHelper = new FsUtility({ + moduleName: 'created-entries', + indexFileName: 'index.json', + basePath: path.join(this.entriesMapperPath, cTUid, locale), + chunkFileSize: this.entriesConfig.chunkFileSize, + keepMetadata: false, + omitKeys: this.entriesConfig.invalidKeys, + }); + const contentType = find(this.cTs, { uid: cTUid }); + + const onSuccess = ({ response, apiData: entry, additionalInfo: { entryFileName } }: any) => { + log(this.importConfig, `Created entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info'); + this.entriesUidMapper[entry.uid] = response.uid; + entry.sourceEntryFilePath = path.join(basePath, entryFileName); // stores source file path temporarily + entry.entryOldUid = entry.uid; // stores old uid temporarily + if (!isMasterLocale) { + this.autoCreatedEntries.push({ cTUid, locale, entryUid: response.uid }); + } + entriesCreateFileHelper.writeIntoFile({ [response.uid]: entry } as any, { mapKeyVal: true }); + }; + const onReject = ({ error, apiData: { uid, title } }: any) => { + log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to create`, 'error'); + log(this.importConfig, formatError(error), 'error'); + this.failedEntries.push({ content_type: cTUid, locale, entry: { uid, title } }); + }; + + for (const index in indexer) { + const chunk = await fs.readChunkFiles.next().catch((error) => { + log(this.importConfig, formatError(error), 'error'); + }); + + if (chunk) { + let apiContent = values(chunk as Record[]); + await this.makeConcurrentCall({ + apiContent, + processName, + indexerCount, + currentIndexer: +index, + apiParams: { + reject: onReject, + resolve: onSuccess, + entity: 'create-entries', + includeParamOnCompletion: true, + serializeData: this.serializeEntries.bind(this), + additionalInfo: { contentType, locale, cTUid, entryFileName: indexer[index] }, + }, + concurrencyLimit: this.importConcurrency, + }).then(() => { + entriesCreateFileHelper?.completeFile(true); + log(this.importConfig, `Created entries for content type ${cTUid} in locale ${locale}`, 'success'); + }); + } + } + } + + /** + * @method serializeEntries + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeEntries(apiOptions: ApiOptions): ApiOptions { + let { + apiData: entry, + additionalInfo: { cTUid, locale, contentType }, + } = apiOptions; + + if (this.jsonRteCTs.indexOf(cTUid) > -1) { + entry = removeUidsFromJsonRteFields(entry, contentType.schema); + } + // remove entry references from json-rte fields + if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) { + entry = removeEntryRefsFromJSONRTE(entry, contentType.schema); + } + // will replace all old asset uid/urls with new ones + entry = lookupAssets( + { + content_type: contentType, + entry: entry, + }, + this.assetUidMapper, + this.assetUrlMapper, + path.join(this.entriesPath, cTUid), + this.installedExtensions, + ); + delete entry.publish_details; + apiOptions.apiData = entry; + return apiOptions; + } + + populateEntryUpdatePayload(): { cTUid: string; locale: string }[] { + const requestOptions: { cTUid: string; locale: string }[] = []; + for (let locale of this.locales) { + for (let cTUid of this.refCTs) { + requestOptions.push({ + cTUid, + locale: locale.code, + }); + } + } + return requestOptions; + } + + async updateEntriesWithReferences({ cTUid, locale }: { cTUid: string; locale: string }): Promise { + const processName = 'Update Entries'; + const indexFileName = 'index.json'; + const basePath = path.join(this.entriesMapperPath, cTUid, locale); + const fs = new FsUtility({ basePath, indexFileName }); + const indexer = fs.indexFileContent; + const indexerCount = values(indexer).length; + if (indexerCount === 0) { + return Promise.resolve(); + } + log(this.importConfig, `Starting to update entries with references for ${cTUid} in locale ${locale}`, 'info'); + + const contentType = find(this.cTs, { uid: cTUid }); + + const onSuccess = ({ response, apiData: { uid, url, title } }: any) => { + log(this.importConfig, `Updated entry: '${title}' of content type ${cTUid} in locale ${locale}`, 'info'); + }; + const onReject = ({ error, apiData: { uid, title } }: any) => { + log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to update`, 'error'); + log(this.importConfig, formatError(error), 'error'); + this.failedEntries.push({ content_type: cTUid, locale, entry: { uid: this.entriesUidMapper[uid], title } }); + }; + + for (const index in indexer) { + const chunk = await fs.readChunkFiles.next().catch((error) => { + log(this.importConfig, formatError(error), 'error'); + }); + + if (chunk) { + let apiContent = values(chunk as Record[]); + await this.makeConcurrentCall({ + apiContent, + processName, + indexerCount, + currentIndexer: +index, + apiParams: { + reject: onReject, + resolve: onSuccess, + entity: 'update-entries', + includeParamOnCompletion: true, + serializeData: this.serializeUpdateEntries.bind(this), + additionalInfo: { contentType, locale, cTUid }, + }, + concurrencyLimit: this.importConcurrency, + }).then(() => { + log(this.importConfig, `Updated entries for content type ${cTUid} in locale ${locale}`, 'success'); + }); + } + } + } + + /** + * @method serializeUpdateEntries + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeUpdateEntries(apiOptions: ApiOptions): ApiOptions { + let { + apiData: entry, + additionalInfo: { cTUid, locale, contentType }, + } = apiOptions; + + try { + const sourceEntryFilePath = entry.sourceEntryFilePath; + const sourceEntry = ((fsUtil.readFile(sourceEntryFilePath) || {}) as Record)[entry.entryOldUid]; + // Removing temp values + delete entry.sourceEntryFilePath; + delete entry.entryOldUid; + if (this.jsonRteCTs.indexOf(cTUid) > -1) { + // the entries stored in eSuccessFilePath, have the same uids as the entries from source data + entry = restoreJsonRteEntryRefs(entry, sourceEntry, contentType.schema, { + mappedAssetUids: this.assetUidMapper, + mappedAssetUrls: this.assetUrlMapper, + }); + } + + entry = lookupEntries( + { + content_type: contentType, + entry, + }, + this.entriesUidMapper, + path.join(this.entriesMapperPath, cTUid, locale), + ); + + const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry.uid]); + Object.assign(entryResponse, cloneDeep(entry)); + delete entryResponse.publish_details; + apiOptions.apiData = entryResponse; + return apiOptions; + } catch (error) { + console.log('error', error); + } + } + + async enableMandatoryCTReferences(): Promise { + const onSuccess = ({ response: contentType, apiData: { uid } }: any) => { + log(this.importConfig, `${uid} content type references updated`, 'success'); + }; + const onReject = ({ error, apiData: { uid } }: any) => { + log(this.importConfig, formatError(error), 'error'); + throw new Error(`Failed to update references of content type ${uid}`); + }; + return await this.makeConcurrentCall({ + processName: 'Update content type references', + apiContent: this.modifiedCTs, + apiParams: { + serializeData: this.serializeUpdateCTsWithRef.bind(this), + reject: onReject.bind(this), + resolve: onSuccess.bind(this), + entity: 'update-cts', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.importConcurrency, + }); + } + + /** + * @method serializeUpdateCTsWithRef + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeUpdateCTsWithRef(apiOptions: ApiOptions): ApiOptions { + const { apiData: contentType } = apiOptions; + if (contentType.field_rules) { + delete contentType.field_rules; + } + lookupExtension( + this.importConfig, + contentType.schema, + this.importConfig.preserveStackVersion, + this.installedExtensions, + ); + const contentTypePayload = this.stack.contentType(contentType.uid); + Object.assign(contentTypePayload, cloneDeep(contentType)); + apiOptions.apiData = contentTypePayload; + return apiOptions; + } + + async removeAutoCreatedEntries(): Promise { + const onSuccess = ({ response, apiData: { entryUid } }: any) => { + log(this.importConfig, `Auto created entry in master locale removed - entry uid ${entryUid} `, 'success'); + }; + const onReject = ({ error, apiData: { entryUid } }: any) => { + log( + this.importConfig, + `Failed to remove auto created entry in master locale - entry uid ${entryUid} \n ${formatError(error)}`, + 'error', + ); + }; + return await this.makeConcurrentCall({ + processName: 'Remove auto created entry in master locale', + apiContent: this.autoCreatedEntries, + apiParams: { + reject: onReject.bind(this), + resolve: onSuccess.bind(this), + entity: 'delete-entries', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.importConcurrency, + }); + } + + async updateFieldRules(): Promise { + let cTsWithFieldRules = fsUtil.readFile(path.join(this.cTsPath + '/field_rules_uid.json')) as Record[]; + if (!cTsWithFieldRules || cTsWithFieldRules?.length === 0) { + return; + } + for (let cTUid of cTsWithFieldRules) { + const contentType = find(this.cTs, { uid: cTUid }); + if (contentType.field_rules) { + let fieldRuleLength = contentType.field_rules.length; + for (let k = 0; k < fieldRuleLength; k++) { + let fieldRuleConditionLength = contentType.field_rules[k].conditions.length; + for (let i = 0; i < fieldRuleConditionLength; i++) { + if (contentType.field_rules[k].conditions[i].operand_field === 'reference') { + let fieldRulesValue = contentType.field_rules[k].conditions[i].value; + let fieldRulesArray = fieldRulesValue.split('.'); + let updatedValue = []; + for (const element of fieldRulesArray) { + let splittedFieldRulesValue = element; + if (this.entriesUidMapper.hasOwnProperty(splittedFieldRulesValue)) { + updatedValue.push(this.entriesUidMapper[splittedFieldRulesValue]); + } else { + updatedValue.push(element); + } + } + contentType.field_rules[k].conditions[i].value = updatedValue.join('.'); + } + } + } + const contentTypeResponse: any = await this.stack + .contentType(contentType.uid) + .fetch() + .catch((error) => { + log(this.importConfig, `failed to update the field rules of ${cTUid} ${formatError(error)}`, 'error'); + }); + if (!contentTypeResponse) { + continue; + } + contentTypeResponse.field_rules = contentType.field_rules; + await contentTypeResponse.update().catch((error: Error) => { + log(this.importConfig, `failed to update the field rules of ${cTUid} ${formatError(error)}`, 'error'); + }); + log(this.importConfig, `Updated the field rules of ${cTUid}`, 'info'); + } else { + log(this.importConfig, `No field rules found in content type ${cTUid} to update`, 'error'); + } + } + } + + async publishEntries({ cTUid, locale }: { cTUid: string; locale: string }): Promise { + const processName = 'Publish Entries'; + const indexFileName = 'index.json'; + const basePath = path.join(this.entriesPath, cTUid, locale); + const fs = new FsUtility({ basePath, indexFileName }); + const indexer = fs.indexFileContent; + const indexerCount = values(indexer).length; + const contentType = find(this.cTs, { uid: cTUid }); + + if (indexerCount === 0) { + return Promise.resolve(); + } + log(this.importConfig, `Starting publish entries for ${cTUid} in locale ${locale}`, 'info'); + + const onSuccess = ({ response, apiData: { environments }, additionalInfo: { entryUid } }: any) => { + log( + this.importConfig, + `Published entry: '${entryUid}' of content type ${cTUid} and locale ${locale} in ${environments?.join( + ',', + )} environments`, + 'info', + ); + }; + const onReject = ({ error, apiData, additionalInfo: { entryUid } }: any) => { + log( + this.importConfig, + `${entryUid} entry of content type ${cTUid} in locale ${locale} failed to publish`, + 'error', + ); + log(this.importConfig, formatError(error), 'error'); + }; + + for (const index in indexer) { + const chunk = await fs.readChunkFiles.next().catch((error) => { + log(this.importConfig, formatError(error), 'error'); + }); + + if (chunk) { + let apiContent = values(chunk as Record[]); + await this.makeConcurrentCall({ + apiContent, + processName, + indexerCount, + currentIndexer: +index, + apiParams: { + reject: onReject, + resolve: onSuccess, + entity: 'publish-entries', + includeParamOnCompletion: true, + serializeData: this.serializePublishEntries.bind(this), + additionalInfo: { contentType, locale, cTUid }, + }, + concurrencyLimit: this.importConcurrency, + }).then(() => { + log(this.importConfig, `Published entries for content type ${cTUid} in locale ${locale}`, 'success'); + }); + } + } + } + + /** + * @method serializeEntries + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializePublishEntries(apiOptions: ApiOptions): ApiOptions { + let { apiData: entry, additionalInfo } = apiOptions; + additionalInfo.entryUid = this.entriesUidMapper[entry.uid]; + const requestObject: { + environments: Array; + locales: Array; + } = { + environments: [], + locales: [], + }; + if (entry.publish_details && entry.publish_details.length > 0) { + forEach(entry.publish_details, (pubObject) => { + if ( + this.envs.hasOwnProperty(pubObject.environment) && + indexOf(requestObject.environments, this.envs[pubObject.environment].name) === -1 + ) { + requestObject.environments.push(this.envs[pubObject.environment].name); + } + if (pubObject.locale && indexOf(requestObject.locales, pubObject.locale) === -1) { + requestObject.locales.push(pubObject.locale); + } + }); + } else { + additionalInfo.skip = true; + } + apiOptions.apiData = requestObject; + return apiOptions; + } +} diff --git a/packages/contentstack-import/src/utils/asset-helper.ts b/packages/contentstack-import/src/utils/asset-helper.ts index 0832824d88..c8fa8cb1ea 100644 --- a/packages/contentstack-import/src/utils/asset-helper.ts +++ b/packages/contentstack-import/src/utils/asset-helper.ts @@ -53,11 +53,11 @@ export const uploadAssetHelper = function (config: ImportConfig, req: any, fsPat // get assets object export const lookupAssets = function ( - data: any, - mappedAssetUids: string[], - mappedAssetUrls: string[], - assetUidMapperPath: string[], - installedExtensions: string[], + data: Record, + mappedAssetUids: Record, + mappedAssetUrls: Record, + assetUidMapperPath: string, + installedExtensions: Record[], ) { if ( !_.has(data, 'entry') || diff --git a/packages/contentstack-import/src/utils/entries-helper.ts b/packages/contentstack-import/src/utils/entries-helper.ts index e432fe01eb..c0d56d42ce 100644 --- a/packages/contentstack-import/src/utils/entries-helper.ts +++ b/packages/contentstack-import/src/utils/entries-helper.ts @@ -8,7 +8,7 @@ import config from '../config'; import * as fileHelper from './file-helper'; // update references in entry object -export const lookupEntries = function (data: any, mappedUids: string[], uidMapperPath: string) { +export const lookupEntries = function (data: any, mappedUids: Record, uidMapperPath: string) { let parent: string[] = []; let uids: string[] = []; let unmapped: string[] = []; @@ -244,3 +244,342 @@ function findUidsInNewRefFields(entry: any, uids: string[]) { } } } + +export const removeUidsFromJsonRteFields = ( + entry: Record, + ctSchema: Record[], +): Record => { + for (const element of ctSchema) { + switch (element.data_type) { + case 'blocks': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e: any) => { + let key = Object.keys(e).pop(); + let subBlock = element.blocks.filter((block: any) => block.uid === key).pop(); + e[key] = removeUidsFromJsonRteFields(e[key], subBlock.schema); + return e; + }); + } + } + break; + } + case 'global_field': + case 'group': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e: any) => { + e = removeUidsFromJsonRteFields(e, element.schema); + return e; + }); + } else { + entry[element.uid] = removeUidsFromJsonRteFields(entry[element.uid], element.schema); + } + } + break; + } + case 'json': { + if (entry[element.uid] && element.field_metadata.rich_text_type) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((jsonRteData: any) => { + delete jsonRteData.uid; // remove uid + + if (_.isObject(jsonRteData.attrs)) { + jsonRteData.attrs.dirty = true; + } + + if (!_.isEmpty(jsonRteData.children)) { + jsonRteData.children = _.map(jsonRteData.children, (child) => removeUidsFromChildren(child)); + } + + return jsonRteData; + }); + } else { + delete entry[element.uid].uid; // remove uid + if (entry[element.uid] && _.isObject(entry[element.uid].attrs)) { + entry[element.uid].attrs.dirty = true; + } + if (entry[element.uid] && !_.isEmpty(entry[element.uid].children)) { + entry[element.uid].children = _.map(entry[element.uid].children, (child) => + removeUidsFromChildren(child), + ); + } + } + } + break; + } + } + } + return entry; +}; + +function removeUidsFromChildren(children: Record[] | any) { + if (children.length && children.length > 0) { + return children.map((child: any) => { + if (child.type && child.type.length > 0) { + delete child.uid; // remove uid + + if (_.isObject(child.attrs)) { + child.attrs.dirty = true; + } + } + if (child.children && child.children.length > 0) { + child.children = removeUidsFromChildren(child.children); + } + return child; + }); + } else { + if (children.type && children.type.length > 0) { + delete children.uid; // remove uid + if (_.isObject(children.attrs)) { + children.attrs.dirty = true; + } + } + if (children.children && children.children.length > 0) { + children.children = removeUidsFromChildren(children.children); + } + return children; + } +} + +export const removeEntryRefsFromJSONRTE = (entry: Record, ctSchema: Record[]) => { + for (const element of ctSchema) { + switch (element.data_type) { + case 'blocks': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e: any) => { + let key = Object.keys(e).pop(); + let subBlock = element.blocks.filter((block: any) => block.uid === key).pop(); + e[key] = removeEntryRefsFromJSONRTE(e[key], subBlock.schema); + return e; + }); + } + } + break; + } + case 'global_field': + case 'group': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e: any) => { + e = removeEntryRefsFromJSONRTE(e, element.schema); + return e; + }); + } else { + entry[element.uid] = removeEntryRefsFromJSONRTE(entry[element.uid], element.schema); + } + } + break; + } + case 'json': { + if (entry[element.uid] && element.field_metadata.rich_text_type) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((jsonRteData: any) => { + // repeated code from else block, will abstract later + let entryReferences = jsonRteData.children.filter((e: any) => doEntryReferencesExist(e)); + if (entryReferences.length > 0) { + jsonRteData.children = jsonRteData.children.filter((e: any) => !doEntryReferencesExist(e)); + return jsonRteData; // return jsonRteData without entry references + } else { + return jsonRteData; // return jsonRteData as it is, because there are no entry references + } + }); + } else { + let entryReferences = entry[element.uid].children.filter((e: any) => doEntryReferencesExist(e)); + if (entryReferences.length > 0) { + entry[element.uid].children = entry[element.uid].children.filter((e: any) => !doEntryReferencesExist(e)); + } + } + } + break; + } + } + } + return entry; +}; + +function doEntryReferencesExist(element: Record[] | any): boolean { + // checks if the children of p element contain any references + // only checking one level deep, not recursive + + if (element.length) { + for (const item of element) { + if ((item.type === 'p' || item.type === 'a') && item.children && item.children.length > 0) { + return doEntryReferencesExist(item.children); + } else if (isEntryRef(item)) { + return true; + } + } + } else { + if (isEntryRef(element)) { + return true; + } + + if ((element.type === 'p' || element.type === 'a') && element.children && element.children.length > 0) { + return doEntryReferencesExist(element.children); + } + } + return false; +} + +function isEntryRef(element: any) { + return element.type === 'reference' && element.attrs.type === 'entry'; +} + +export const restoreJsonRteEntryRefs = ( + entry: Record, + sourceStackEntry: any, + ctSchema: any, + { mappedAssetUids, mappedAssetUrls }: any, +) => { + // let mappedAssetUids = fileHelper.readFileSync(this.mappedAssetUidPath) || {}; + // let mappedAssetUrls = fileHelper.readFileSync(this.mappedAssetUrlPath) || {}; + for (const element of ctSchema) { + switch (element.data_type) { + case 'blocks': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e: any, eIndex: number) => { + let key = Object.keys(e).pop(); + let subBlock = element.blocks.filter((block: any) => block.uid === key).pop(); + let sourceStackElement = sourceStackEntry[element.uid][eIndex][key]; + e[key] = restoreJsonRteEntryRefs(e[key], sourceStackElement, subBlock.schema, { + mappedAssetUids, + mappedAssetUrls, + }); + return e; + }); + } + } + break; + } + case 'global_field': + case 'group': { + if (entry[element.uid]) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((e: any, eIndex: number) => { + let sourceStackElement = sourceStackEntry[element.uid][eIndex]; + e = restoreJsonRteEntryRefs(e, sourceStackElement, element.schema, { mappedAssetUids, mappedAssetUrls }); + return e; + }); + } else { + let sourceStackElement = sourceStackEntry[element.uid]; + entry[element.uid] = restoreJsonRteEntryRefs(entry[element.uid], sourceStackElement, element.schema, { + mappedAssetUids, + mappedAssetUrls, + }); + } + } + break; + } + case 'json': { + if (entry[element.uid] && element.field_metadata.rich_text_type) { + if (element.multiple) { + entry[element.uid] = entry[element.uid].map((field: any, index: number) => { + // i am facing a Maximum call stack exceeded issue, + // probably because of this loop operation + + let entryRefs = sourceStackEntry[element.uid][index].children + .map((e: any, i: number) => { + return { index: i, value: e }; + }) + .filter((e: any) => doEntryReferencesExist(e.value)) + .map((e: any) => { + // commenting the line below resolved the maximum call stack exceeded issue + // e.value = this.setDirtyTrue(e.value) + setDirtyTrue(e.value); + return e; + }) + .map((e: any) => { + // commenting the line below resolved the maximum call stack exceeded issue + // e.value = this.resolveAssetRefsInEntryRefsForJsonRte(e, mappedAssetUids, mappedAssetUrls) + resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls); + return e; + }); + + if (entryRefs.length > 0) { + entryRefs.forEach((entryRef: any) => { + field.children.splice(entryRef.index, 0, entryRef.value); + }); + } + return field; + }); + } else { + let entryRefs = sourceStackEntry[element.uid].children + .map((e: any, index: number) => { + return { index: index, value: e }; + }) + .filter((e: any) => doEntryReferencesExist(e.value)) + .map((e: any) => { + setDirtyTrue(e.value); + return e; + }) + .map((e: any) => { + resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls); + return e; + }); + + if (entryRefs.length > 0) { + entryRefs.forEach((entryRef: any) => { + if (!_.isEmpty(entry[element.uid]) && entry[element.uid].children) { + entry[element.uid].children.splice(entryRef.index, 0, entryRef.value); + } + }); + } + } + } + break; + } + } + } + return entry; +}; + +function setDirtyTrue(jsonRteChild: any) { + // also removing uids in this function + if (jsonRteChild.type) { + if (_.isObject(jsonRteChild.attrs)) { + jsonRteChild.attrs['dirty'] = true; + } + delete jsonRteChild.uid; + + if (jsonRteChild.children && jsonRteChild.children.length > 0) { + jsonRteChild.children = jsonRteChild.children.map((subElement: any) => this.setDirtyTrue(subElement)); + } + } + return jsonRteChild; +} + +function resolveAssetRefsInEntryRefsForJsonRte(jsonRteChild: any, mappedAssetUids: any, mappedAssetUrls: any) { + if (jsonRteChild.type) { + if (jsonRteChild.attrs.type === 'asset') { + let assetUrl; + if (mappedAssetUids[jsonRteChild.attrs['asset-uid']]) { + jsonRteChild.attrs['asset-uid'] = mappedAssetUids[jsonRteChild.attrs['asset-uid']]; + } + + if (jsonRteChild.attrs['display-type'] !== 'link') { + assetUrl = jsonRteChild.attrs['asset-link']; + } else { + assetUrl = jsonRteChild.attrs['href']; + } + + if (mappedAssetUrls[assetUrl]) { + if (jsonRteChild.attrs['display-type'] !== 'link') { + jsonRteChild.attrs['asset-link'] = mappedAssetUrls[assetUrl]; + } else { + jsonRteChild.attrs['href'] = mappedAssetUrls[assetUrl]; + } + } + } + + if (jsonRteChild.children && jsonRteChild.children.length > 0) { + jsonRteChild.children = jsonRteChild.children.map((subElement: any) => + resolveAssetRefsInEntryRefsForJsonRte(subElement, mappedAssetUids, mappedAssetUrls), + ); + } + } + + return jsonRteChild; +} diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index 138e589111..19cfca611e 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -16,9 +16,14 @@ export { confirmToCloseProcess, getAllStackSpecificApps, ifAppAlreadyExist, - updateAppConfig + updateAppConfig, } from './marketplace-app-helper'; export { schemaTemplate, suppressSchemaReference, removeReferenceFields } from './content-type-helper'; export { lookupExtension } from './extension-helper'; -export { lookupEntries } from './entries-helper'; +export { + lookupEntries, + removeUidsFromJsonRteFields, + removeEntryRefsFromJSONRTE, + restoreJsonRteEntryRefs, +} from './entries-helper'; export * from './common-helper';