diff --git a/packages/contentstack-auth/bin/dev b/packages/contentstack-auth/bin/dev index f003f9d180..39382a0159 100644 --- a/packages/contentstack-auth/bin/dev +++ b/packages/contentstack-auth/bin/dev @@ -2,5 +2,5 @@ (async () => { const { execute } = require('@contentstack/cli-utilities'); - await execute({ type: 'cjs', development: true, dir: __dirname }); -})(); \ No newline at end of file + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); diff --git a/packages/contentstack-bootstrap/bin/dev b/packages/contentstack-bootstrap/bin/dev new file mode 100644 index 0000000000..39382a0159 --- /dev/null +++ b/packages/contentstack-bootstrap/bin/dev @@ -0,0 +1,6 @@ +#!/usr/bin/env node + +(async () => { + const { execute } = require('@contentstack/cli-utilities'); + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); diff --git a/packages/contentstack-bootstrap/bin/dev.cmd b/packages/contentstack-bootstrap/bin/dev.cmd new file mode 100644 index 0000000000..8ae2b12c19 --- /dev/null +++ b/packages/contentstack-bootstrap/bin/dev.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\dev" %* diff --git a/packages/contentstack-config/bin/dev b/packages/contentstack-config/bin/dev new file mode 100644 index 0000000000..39382a0159 --- /dev/null +++ b/packages/contentstack-config/bin/dev @@ -0,0 +1,6 @@ +#!/usr/bin/env node + +(async () => { + const { execute } = require('@contentstack/cli-utilities'); + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); diff --git a/packages/contentstack-config/bin/dev.cmd b/packages/contentstack-config/bin/dev.cmd new file mode 100644 index 0000000000..8ae2b12c19 --- /dev/null +++ b/packages/contentstack-config/bin/dev.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\dev" %* diff --git a/packages/contentstack-export/bin/dev b/packages/contentstack-export/bin/dev index 74555287cd..59b22a63af 100755 --- a/packages/contentstack-export/bin/dev +++ b/packages/contentstack-export/bin/dev @@ -1,4 +1,5 @@ #!/usr/bin/env node + (async () => { const oclif = require('@oclif/core'); await oclif.execute({ type: 'cjs', development: true, dir: __dirname }); diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index fb12f2f075..9a98fdcc8a 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -96,8 +96,8 @@ const config: DefaultConfig = { invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by', '_metadata', 'published'], // no of asset version files (of a single asset) that'll be downloaded parallel chunkFileSize: 1, // measured on Megabits (5mb) - downloadLimit: 10, - fetchConcurrency: 10, + downloadLimit: 5, + fetchConcurrency: 5, assetsMetaKeys: [], // Default keys ['uid', 'url', 'filename'] securedAssets: false, displayExecutionTime: false, diff --git a/packages/contentstack-export/src/export/modules/entries.ts b/packages/contentstack-export/src/export/modules/entries.ts index f795cee78c..b1981929cf 100644 --- a/packages/contentstack-export/src/export/modules/entries.ts +++ b/packages/contentstack-export/src/export/modules/entries.ts @@ -54,11 +54,11 @@ export default class EntriesExport extends BaseClass { } const entryRequestOptions = this.createRequestObjects(locales, contentTypes); for (let entryRequestOption of entryRequestOptions) { - log( - this.exportConfig, - `Starting export of entries of content type - ${entryRequestOption.contentType} locale - ${entryRequestOption.locale}`, - 'info', - ); + // log( + // this.exportConfig, + // `Starting export of entries of content type - ${entryRequestOption.contentType} locale - ${entryRequestOption.locale}`, + // 'info', + // ); await this.getEntries(entryRequestOption); this.entriesFileHelper?.completeFile(true); log( diff --git a/packages/contentstack-export/tsconfig.json b/packages/contentstack-export/tsconfig.json index 5136623392..da8fc4b724 100644 --- a/packages/contentstack-export/tsconfig.json +++ b/packages/contentstack-export/tsconfig.json @@ -12,7 +12,7 @@ "skipLibCheck": true, "sourceMap": false, "esModuleInterop": true, - "noImplicitAny": true, + "noImplicitAny": false, "lib": [ "ES2019", "es2020.promise" diff --git a/packages/contentstack-import/bin/dev b/packages/contentstack-import/bin/dev index 02b50d62ab..59b22a63af 100644 --- a/packages/contentstack-import/bin/dev +++ b/packages/contentstack-import/bin/dev @@ -1,17 +1,6 @@ #!/usr/bin/env node -const oclif = require('@oclif/core'); - -const path = require('path'); -const project = path.join(__dirname, '..', 'tsconfig.json'); - -// In dev mode -> use ts-node and dev plugins -process.env.NODE_ENV = 'development'; - -require('ts-node').register({ project }); - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle); +(async () => { + const oclif = require('@oclif/core'); + await oclif.execute({ type: 'cjs', development: true, dir: __dirname }); +})(); diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 4a5621fd05..d1745a1288 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -84,9 +84,9 @@ const config: DefaultConfig = { publishAssets: true, fileName: 'assets.json', importSameStructure: true, - uploadAssetsConcurrency: 10, + uploadAssetsConcurrency: 5, displayExecutionTime: false, - importFoldersConcurrency: 10, + importFoldersConcurrency: 1, includeVersionedAssets: false, host: 'https://api.contentstack.io', folderValidKeys: ['name', 'parent_uid'], diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index 1863bed5a8..8b048da5bf 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -4,6 +4,7 @@ import filter from 'lodash/filter'; import unionBy from 'lodash/unionBy'; import orderBy from 'lodash/orderBy'; import isEmpty from 'lodash/isEmpty'; +import uniq from 'lodash/uniq'; import { existsSync } from 'node:fs'; import includes from 'lodash/includes'; import { resolve as pResolve, join } from 'node:path'; @@ -258,9 +259,11 @@ export default class ImportAssets extends BaseClass { }; const serializeData = (apiOptions: ApiOptions) => { const { apiData: asset } = apiOptions; - const publishDetails = filter(asset.publish_details, 'environment'); + const publishDetails = filter(asset.publish_details, ({ environment }) => { + return this.environments.hasOwnProperty(environment); + }); + const environments = uniq(map(publishDetails, ({ environment }) => this.environments[environment].name)); const locales = map(publishDetails, 'locale'); - const environments = map(publishDetails, ({ environment }) => this.environments[environment].name); asset.locales = locales; asset.environments = environments; diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 932e375235..3510d8ae88 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -299,8 +299,8 @@ export default abstract class BaseClass { case 'create-cts': return this.stack.contentType().create(apiData).then(onSuccess).catch(onReject); case 'update-cts': - if (additionalInfo.skip) { - return Promise.resolve(onSuccess(apiData)); + if (!apiData) { + return Promise.resolve(); } return apiData.update().then(onSuccess).catch(onReject); case 'update-gfs': @@ -345,6 +345,9 @@ export default abstract class BaseClass { .then(onSuccess) .catch(onReject); case 'create-entries': + if (additionalInfo[apiData?.uid]?.isLocalized) { + return apiData.update({ locale: additionalInfo.locale }).then(onSuccess).catch(onReject); + } return this.stack .contentType(additionalInfo.cTUid) .entry() @@ -354,20 +357,23 @@ export default abstract class BaseClass { case 'update-entries': return apiData.update({ locale: additionalInfo.locale }).then(onSuccess).catch(onReject); case 'publish-entries': - if (additionalInfo.skip) { - return Promise.resolve(onSuccess(apiData)); + if (!apiData) { + return Promise.resolve(); } return this.stack .contentType(additionalInfo.cTUid) - .entry(additionalInfo.entryUid) - .publish({ publishDetails: apiData, locale: additionalInfo.locale }) + .entry(apiData.entryUid) + .publish({ + publishDetails: { environments: apiData.environments, locales: apiData.locales }, + locale: additionalInfo.locale, + }) .then(onSuccess) .catch(onReject); case 'delete-entries': return this.stack .contentType(apiData.cTUid) .entry(apiData.entryUid) - .delete({ locale: this.importConfig?.master_locale?.code }) + .delete({ locale: additionalInfo.locale }) .then(onSuccess) .catch(onReject); default: diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index cf84edfe08..1188d18f90 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -80,6 +80,7 @@ export default class EntriesImport extends BaseClass { this.jsonRteCTsWithRef = []; this.envs = {}; this.autoCreatedEntries = []; + this.failedEntries = []; } async start(): Promise { @@ -109,6 +110,17 @@ export default class EntriesImport extends BaseClass { await fileHelper.writeLargeFile(path.join(this.entriesMapperPath, 'uid-mapping.json'), this.entriesUidMapper); // TBD: manages mapper in one file, should find an alternative fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries); + if (this.autoCreatedEntries.length > 0) { + log(this.importConfig, 'Removing entries from master language which got created by default', 'info'); + await this.removeAutoCreatedEntries().catch((error) => { + log( + this.importConfig, + `Error while removing auto created entries in master locale ${formatError(error)}`, + 'error', + ); + }); + } + // Update entries with references const entryUpdateRequestOptions = this.populateEntryUpdatePayload(); for (let entryUpdateRequestOption of entryUpdateRequestOptions) { @@ -128,16 +140,6 @@ export default class EntriesImport extends BaseClass { log(this.importConfig, `Error while updating content type references ${formatError(error)}`, 'error'); }); - if (this.autoCreatedEntries.length > 0) { - log(this.importConfig, 'Removing entries from master language which got created by default', 'info'); - await this.removeAutoCreatedEntries().catch((error) => { - log( - this.importConfig, - `Error while removing auto created entries in master locale ${formatError(error)}`, - 'error', - ); - }); - } // Update field rule of content types which are got removed earlier log(this.importConfig, 'Updating the field rules of content type', 'info'); await this.updateFieldRules().catch((error) => { @@ -209,14 +211,6 @@ export default class EntriesImport extends BaseClass { jsonRteEmbeddedEntries: false, }; suppressSchemaReference(contentType.schema, flag); - // Check if suppress modified flag - if (flag.suppressed) { - this.modifiedCTs.push(find(this.cTs, { uid: contentType.uid })); - } else { - // Note: Skips the content type from update if no reference found - apiOptions.additionalInfo = { skip: true }; - return apiOptions; - } if (flag.references) { this.refCTs.push(contentType.uid); @@ -231,6 +225,16 @@ export default class EntriesImport extends BaseClass { } } } + + // Check if suppress modified flag + if (flag.suppressed) { + this.modifiedCTs.push(find(this.cTs, { uid: contentType.uid })); + } else { + // Note: Skips the content type from update if no reference found + apiOptions.apiData = null; + return apiOptions; + } + lookupExtension( this.importConfig, contentType.schema, @@ -266,11 +270,11 @@ export default class EntriesImport extends BaseClass { if (indexerCount === 0) { return Promise.resolve(); } - log(this.importConfig, `Starting to create entries for ${cTUid} in locale ${locale}`, 'info'); + // log(this.importConfig, `Starting to create entries for ${cTUid} in locale ${locale}`, 'info'); const isMasterLocale = locale === this.importConfig?.master_locale?.code; // Write created entries const entriesCreateFileHelper = new FsUtility({ - moduleName: 'created-entries', + moduleName: 'entries', indexFileName: 'index.json', basePath: path.join(this.entriesMapperPath, cTUid, locale), chunkFileSize: this.entriesConfig.chunkFileSize, @@ -279,15 +283,15 @@ export default class EntriesImport extends BaseClass { }); const contentType = find(this.cTs, { uid: cTUid }); - const onSuccess = ({ response, apiData: entry, additionalInfo: { entryFileName } }: any) => { + const onSuccess = ({ response, apiData: entry, additionalInfo }: any) => { log(this.importConfig, `Created entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info'); - this.entriesUidMapper[entry.uid] = response.uid; - entry.sourceEntryFilePath = path.join(basePath, entryFileName); // stores source file path temporarily - entry.entryOldUid = entry.uid; // stores old uid temporarily - if (!isMasterLocale) { + if (!isMasterLocale && !additionalInfo[entry.uid]?.isLocalized) { this.autoCreatedEntries.push({ cTUid, locale, entryUid: response.uid }); } - entriesCreateFileHelper.writeIntoFile({ [response.uid]: entry } as any, { mapKeyVal: true }); + this.entriesUidMapper[entry.uid] = response.uid; + entry.sourceEntryFilePath = path.join(basePath, additionalInfo.entryFileName); // stores source file path temporarily + entry.entryOldUid = entry.uid; // stores old uid temporarily + entriesCreateFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true }); }; const onReject = ({ error, apiData: { uid, title } }: any) => { log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to create`, 'error'); @@ -313,7 +317,7 @@ export default class EntriesImport extends BaseClass { entity: 'create-entries', includeParamOnCompletion: true, serializeData: this.serializeEntries.bind(this), - additionalInfo: { contentType, locale, cTUid, entryFileName: indexer[index] }, + additionalInfo: { contentType, locale, cTUid, entryFileName: indexer[index], isMasterLocale }, }, concurrencyLimit: this.importConcurrency, }).then(() => { @@ -332,7 +336,7 @@ export default class EntriesImport extends BaseClass { serializeEntries(apiOptions: ApiOptions): ApiOptions { let { apiData: entry, - additionalInfo: { cTUid, locale, contentType }, + additionalInfo: { cTUid, locale, contentType, isMasterLocale }, } = apiOptions; if (this.jsonRteCTs.indexOf(cTUid) > -1) { @@ -354,6 +358,16 @@ export default class EntriesImport extends BaseClass { this.installedExtensions, ); delete entry.publish_details; + // checking the entry is a localized one or not + if (!isMasterLocale && this.entriesUidMapper.hasOwnProperty(entry.uid)) { + const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry.uid]); + Object.assign(entryResponse, cloneDeep(entry), { uid: this.entriesUidMapper[entry.uid] }); + apiOptions.apiData = entryResponse; + apiOptions.additionalInfo[entryResponse.uid] = { + isLocalized: true, + }; + return apiOptions; + } apiOptions.apiData = entry; return apiOptions; } @@ -381,7 +395,7 @@ export default class EntriesImport extends BaseClass { if (indexerCount === 0) { return Promise.resolve(); } - log(this.importConfig, `Starting to update entries with references for ${cTUid} in locale ${locale}`, 'info'); + // log(this.importConfig, `Starting to update entries with references for ${cTUid} in locale ${locale}`, 'info'); const contentType = find(this.cTs, { uid: cTUid }); @@ -391,7 +405,12 @@ export default class EntriesImport extends BaseClass { const onReject = ({ error, apiData: { uid, title } }: any) => { log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to update`, 'error'); log(this.importConfig, formatError(error), 'error'); - this.failedEntries.push({ content_type: cTUid, locale, entry: { uid: this.entriesUidMapper[uid], title } }); + this.failedEntries.push({ + content_type: cTUid, + locale, + entry: { uid: this.entriesUidMapper[uid], title }, + entryId: uid, + }); }; for (const index in indexer) { @@ -456,7 +475,7 @@ export default class EntriesImport extends BaseClass { ); const entryResponse = this.stack.contentType(contentType.uid).entry(this.entriesUidMapper[entry.uid]); - Object.assign(entryResponse, cloneDeep(entry)); + Object.assign(entryResponse, cloneDeep(entry), { uid: this.entriesUidMapper[entry.uid] }); delete entryResponse.publish_details; apiOptions.apiData = entryResponse; return apiOptions; @@ -525,6 +544,7 @@ export default class EntriesImport extends BaseClass { resolve: onSuccess.bind(this), entity: 'delete-entries', includeParamOnCompletion: true, + additionalInfo: { locale: this.importConfig?.master_locale?.code }, }, concurrencyLimit: this.importConcurrency, }); @@ -590,9 +610,9 @@ export default class EntriesImport extends BaseClass { if (indexerCount === 0) { return Promise.resolve(); } - log(this.importConfig, `Starting publish entries for ${cTUid} in locale ${locale}`, 'info'); + // log(this.importConfig, `Starting publish entries for ${cTUid} in locale ${locale}`, 'info'); - const onSuccess = ({ response, apiData: { environments }, additionalInfo: { entryUid } }: any) => { + const onSuccess = ({ response, apiData: { environments, entryUid }, additionalInfo }: any) => { log( this.importConfig, `Published entry: '${entryUid}' of content type ${cTUid} and locale ${locale} in ${environments?.join( @@ -601,10 +621,10 @@ export default class EntriesImport extends BaseClass { 'info', ); }; - const onReject = ({ error, apiData, additionalInfo: { entryUid } }: any) => { + const onReject = ({ error, apiData, additionalInfo }: any) => { log( this.importConfig, - `${entryUid} entry of content type ${cTUid} in locale ${locale} failed to publish`, + `${apiData.entryUid} entry of content type ${cTUid} in locale ${locale} failed to publish`, 'error', ); log(this.importConfig, formatError(error), 'error'); @@ -645,13 +665,14 @@ export default class EntriesImport extends BaseClass { */ serializePublishEntries(apiOptions: ApiOptions): ApiOptions { let { apiData: entry, additionalInfo } = apiOptions; - additionalInfo.entryUid = this.entriesUidMapper[entry.uid]; const requestObject: { environments: Array; locales: Array; + entryUid: string; } = { environments: [], locales: [], + entryUid: this.entriesUidMapper[entry.uid], }; if (entry.publish_details && entry.publish_details.length > 0) { forEach(entry.publish_details, (pubObject) => { @@ -666,7 +687,8 @@ export default class EntriesImport extends BaseClass { } }); } else { - additionalInfo.skip = true; + apiOptions.apiData = null; + return apiOptions; } apiOptions.apiData = requestObject; return apiOptions; diff --git a/packages/contentstack-import/src/import/modules/global-fields.ts b/packages/contentstack-import/src/import/modules/global-fields.ts index c0fe880c30..958ada0c17 100644 --- a/packages/contentstack-import/src/import/modules/global-fields.ts +++ b/packages/contentstack-import/src/import/modules/global-fields.ts @@ -116,8 +116,9 @@ export default class ImportGlobalFields extends BaseClass { }) { return new Promise(async (resolve, reject) => { lookupExtension(this.config, globalField.schema, this.config.preserveStackVersion, this.installedExtensions); - const isReferenceFieldRemoved = await removeReferenceFields(globalField.schema, undefined, this.stackAPIClient); - if (isReferenceFieldRemoved) { + let flag = { supressed: false }; + await removeReferenceFields(globalField.schema, flag, this.stackAPIClient); + if (flag.supressed) { this.pendingGFs.push(globalField.uid); } return this.stack diff --git a/packages/contentstack-import/src/utils/backup-handler.ts b/packages/contentstack-import/src/utils/backup-handler.ts index 802a727896..bd76abf784 100755 --- a/packages/contentstack-import/src/utils/backup-handler.ts +++ b/packages/contentstack-import/src/utils/backup-handler.ts @@ -17,9 +17,12 @@ export default function setupBackupDir(importConfig: ImportConfig): Promise use ts-node and dev plugins -process.env.NODE_ENV = 'development' - -require('ts-node').register({project }) - -// In dev mode, always show stack traces -oclif.settings.debug = true; - -// Start the CLI -oclif.run().then(oclif.flush).catch(oclif.Errors.handle) +(async () => { + const { execute } = require('@contentstack/cli-utilities'); + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); diff --git a/packages/contentstack-seed/bin/dev b/packages/contentstack-seed/bin/dev new file mode 100644 index 0000000000..39382a0159 --- /dev/null +++ b/packages/contentstack-seed/bin/dev @@ -0,0 +1,6 @@ +#!/usr/bin/env node + +(async () => { + const { execute } = require('@contentstack/cli-utilities'); + await execute({ type: 'cjs', dir: __dirname, development: true }); +})(); diff --git a/packages/contentstack-seed/bin/dev.cmd b/packages/contentstack-seed/bin/dev.cmd new file mode 100644 index 0000000000..8ae2b12c19 --- /dev/null +++ b/packages/contentstack-seed/bin/dev.cmd @@ -0,0 +1,3 @@ +@echo off + +node "%~dp0\dev" %* diff --git a/packages/contentstack-utilities/src/fs-utility/core.ts b/packages/contentstack-utilities/src/fs-utility/core.ts index 13d796d77d..dcfaea4ad8 100644 --- a/packages/contentstack-utilities/src/fs-utility/core.ts +++ b/packages/contentstack-utilities/src/fs-utility/core.ts @@ -285,8 +285,8 @@ export default class FsUtility { if (this.fileExt === 'json') { this.writableStream.write('}'); } - this.closeFile(closeIndexer); } + this.closeFile(closeIndexer); } /**