Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
entries overwrite feature
Browse files Browse the repository at this point in the history
Shafeeq PP authored and Shafeeq PP committed Oct 4, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent ec5d28a commit 1b44c1f
Showing 7 changed files with 207 additions and 28 deletions.
178 changes: 172 additions & 6 deletions packages/contentstack-import/src/import/modules/entries.ts
Original file line number Diff line number Diff line change
@@ -107,6 +107,15 @@ export default class EntriesImport extends BaseClass {
for (let entryRequestOption of entryRequestOptions) {
await this.createEntries(entryRequestOption);
}
if (this.importConfig.replaceExisting) {
// Note: Instead of using entryRequestOptions, we can prepare request options for replace, to avoid unnecessary operations
for (let entryRequestOption of entryRequestOptions) {
await this.replaceEntries(entryRequestOption).catch((error) => {
log(this.importConfig, `Error while replacing the existing entries ${formatError(error)}`, 'error');
});
}
}

await fileHelper.writeLargeFile(path.join(this.entriesMapperPath, 'uid-mapping.json'), this.entriesUidMapper); // TBD: manages mapper in one file, should find an alternative
fsUtil.writeFile(path.join(this.entriesMapperPath, 'failed-entries.json'), this.failedEntries);

@@ -281,12 +290,27 @@ export default class EntriesImport extends BaseClass {
keepMetadata: false,
omitKeys: this.entriesConfig.invalidKeys,
});

// create file instance for existing entries
const existingEntriesFileHelper = new FsUtility({
moduleName: 'entries',
indexFileName: 'index.json',
basePath: path.join(this.entriesMapperPath, cTUid, locale, 'existing'),
chunkFileSize: this.entriesConfig.chunkFileSize,
keepMetadata: false,
omitKeys: this.entriesConfig.invalidKeys,
});

const contentType = find(this.cTs, { uid: cTUid });

const onSuccess = ({ response, apiData: entry, additionalInfo }: any) => {
if (additionalInfo[entry.uid]?.isLocalized) {
let oldUid = additionalInfo[entry.uid].entryOldUid;
log(this.importConfig, `Localized entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info');
log(
this.importConfig,
`Localized entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`,
'info',
);
entry.uid = oldUid;
entry.entryOldUid = oldUid;
entry.sourceEntryFilePath = path.join(basePath, additionalInfo.entryFileName); // stores source file path temporarily
@@ -306,10 +330,23 @@ export default class EntriesImport extends BaseClass {
entriesCreateFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true });
}
};
const onReject = ({ error, apiData: { uid, title } }: any) => {
log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to create`, 'error');
log(this.importConfig, formatError(error), 'error');
this.failedEntries.push({ content_type: cTUid, locale, entry: { uid, title } });
const onReject = ({ error, apiData: entry, additionalInfo }: any) => {
const { title, uid } = entry;
//Note: write existing entries into files to handler later
if (error?.errors?.title) {
if (this.importConfig.replaceExisting) {
entry.entryOldUid = uid;
entry.sourceEntryFilePath = path.join(basePath, additionalInfo.entryFileName); // stores source file path temporarily
existingEntriesFileHelper.writeIntoFile({ [uid]: entry } as any, { mapKeyVal: true });
}
if (!this.importConfig.skipExisting) {
log(this.importConfig, `Entry '${title}' already exists`, 'info');
}
} else {
log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to create`, 'error');
log(this.importConfig, formatError(error), 'error');
this.failedEntries.push({ content_type: cTUid, locale, entry: { uid, title } });
}
};

for (const index in indexer) {
@@ -335,6 +372,7 @@ export default class EntriesImport extends BaseClass {
concurrencyLimit: this.importConcurrency,
}).then(() => {
entriesCreateFileHelper?.completeFile(true);
existingEntriesFileHelper?.completeFile(true);
log(this.importConfig, `Created entries for content type ${cTUid} in locale ${locale}`, 'success');
});
}
@@ -379,7 +417,7 @@ export default class EntriesImport extends BaseClass {
apiOptions.apiData = entryResponse;
apiOptions.additionalInfo[entryResponse.uid] = {
isLocalized: true,
entryOldUid: entry.uid
entryOldUid: entry.uid,
};
return apiOptions;
}
@@ -397,6 +435,134 @@ export default class EntriesImport extends BaseClass {
return apiOptions;
}

async replaceEntries({ cTUid, locale }: { cTUid: string; locale: string }): Promise<void> {
const processName = 'Replace existing Entries';
const indexFileName = 'index.json';
const basePath = path.join(this.entriesMapperPath, cTUid, locale, 'existing');
const fs = new FsUtility({ basePath, indexFileName });
const indexer = fs.indexFileContent;
const indexerCount = values(indexer).length;
if (indexerCount === 0) {
return Promise.resolve();
}

// Write updated entries
const entriesReplaceFileHelper = new FsUtility({
moduleName: 'entries',
indexFileName: 'index.json',
basePath: path.join(this.entriesMapperPath, cTUid, locale),
chunkFileSize: this.entriesConfig.chunkFileSize,
keepMetadata: false,
useIndexer: true,
omitKeys: this.entriesConfig.invalidKeys,
});
// log(this.importConfig, `Starting to update entries with references for ${cTUid} in locale ${locale}`, 'info');

const contentType = find(this.cTs, { uid: cTUid });

const onSuccess = ({ response, apiData: entry, additionalInfo }: any) => {
log(this.importConfig, `Replaced entry: '${entry.title}' of content type ${cTUid} in locale ${locale}`, 'info');
this.entriesUidMapper[entry.uid] = response.uid;
entriesReplaceFileHelper.writeIntoFile({ [entry.uid]: entry } as any, { mapKeyVal: true });
};
const onReject = ({ error, apiData: { uid, title } }: any) => {
log(this.importConfig, `${title} entry of content type ${cTUid} in locale ${locale} failed to replace`, 'error');
log(this.importConfig, formatError(error), 'error');
this.failedEntries.push({
content_type: cTUid,
locale,
entry: { uid: this.entriesUidMapper[uid], title },
entryId: uid,
});
};

for (const index in indexer) {
const chunk = await fs.readChunkFiles.next().catch((error) => {
log(this.importConfig, formatError(error), 'error');
});

if (chunk) {
let apiContent = values(chunk as Record<string, any>[]);
await this.makeConcurrentCall(
{
apiContent,
processName,
indexerCount,
currentIndexer: +index,
apiParams: {
reject: onReject,
resolve: onSuccess,
entity: 'update-entries',
includeParamOnCompletion: true,
additionalInfo: { contentType, locale, cTUid },
},
concurrencyLimit: this.importConcurrency,
},
this.replaceEntriesHandler.bind(this),
).then(() => {
entriesReplaceFileHelper?.completeFile(true);
log(this.importConfig, `Replaced entries for content type ${cTUid} in locale ${locale}`, 'success');
});
}
}
}

async replaceEntriesHandler({
apiParams,
element: entry,
isLastRequest,
}: {
apiParams: ApiOptions;
element: Record<string, string>;
isLastRequest: boolean;
}) {
const { additionalInfo: { cTUid, locale } = {} } = apiParams;
return new Promise(async (resolve, reject) => {
const { items: [entryInStack] = [] }: any = await this.stack
.contentType(cTUid)
.entry()
.query({ query: { title: entry.title, locale } })
.findOne()
.catch((error: Error) => {
apiParams.reject({
error,
apiData: entry,
});
reject(true);
});
if (entryInStack) {
const entryPayload = this.stack.contentType(cTUid).entry(entryInStack.uid);
Object.assign(entryPayload, entryInStack, cloneDeep(entry), {
uid: entryInStack.uid,
urlPath: entryInStack.urlPath,
stackHeaders: entryInStack.stackHeaders,
_version: entryInStack._version,
});
return entryPayload
.update()
.then((response: any) => {
apiParams.resolve({
response,
apiData: entry,
});
resolve(true);
})
.catch((error: Error) => {
apiParams.reject({
error,
apiData: entry,
});
reject(true);
});
} else {
apiParams.reject({
error: new Error(`Extension with title ${entry.title} not found in the stack`),
apiData: entry,
});
reject(true);
}
});
}
populateEntryUpdatePayload(): { cTUid: string; locale: string }[] {
const requestOptions: { cTUid: string; locale: string }[] = [];
for (let locale of this.locales) {
Original file line number Diff line number Diff line change
@@ -178,9 +178,11 @@ export default class ImportExtensions extends BaseClass {
});
if (extensionsInStack) {
const extensionPayload = this.stack.extension(extension.uid);
Object.assign(extensionPayload, cloneDeep(extensionsInStack), {
Object.assign(extensionPayload, extensionsInStack, cloneDeep(extension), {
uid: extensionsInStack.uid,
urlPath: extensionsInStack.urlPath,
_version: extensionsInStack._version,
stackHeaders: extensionsInStack.stackHeaders,
});
return extensionPayload
.update()
10 changes: 6 additions & 4 deletions packages/contentstack-import/src/import/modules/global-fields.ts
Original file line number Diff line number Diff line change
@@ -97,9 +97,9 @@ export default class ImportGlobalFields extends BaseClass {
if (error?.errors?.title) {
if (this.importConfig.replaceExisting) {
// Note: skipping the pending GFs from update, reason: pending GFs gets updated at the CTs import
if (this.pendingGFs.indexOf(uid) === -1) {
this.existingGFs.push(globalField);
}
// if (this.pendingGFs.indexOf(uid) === -1) {
this.existingGFs.push(globalField);
// }
}
if (!this.importConfig.skipExisting) {
log(this.importConfig, `Global fields '${uid}' already exist`, 'info');
@@ -203,7 +203,9 @@ export default class ImportGlobalFields extends BaseClass {
serializeReplaceGFs(apiOptions: ApiOptions): ApiOptions {
const { apiData: globalField } = apiOptions;
const globalFieldPayload = this.stack.globalField(globalField.uid);
Object.assign(globalFieldPayload, cloneDeep(globalField));
Object.assign(globalFieldPayload, cloneDeep(globalField), {
stackHeaders: globalFieldPayload.stackHeaders,
});
apiOptions.apiData = globalFieldPayload;
return apiOptions;
}
11 changes: 8 additions & 3 deletions packages/contentstack-import/src/import/modules/locales.ts
Original file line number Diff line number Diff line change
@@ -137,9 +137,14 @@ export default class ImportLocales extends BaseClass {
fsUtil.writeFile(this.langUidMapperPath, this.langUidMapper);
};
const onReject = ({ error, apiData: { uid, code } = undefined }: any) => {
log(this.importConfig, `Language '${code}' failed to import`, 'error');
log(this.importConfig, formatError(error), 'error');
this.failedLocales.push({ uid, code });
if (error?.errorCode === 247) {
log(this.importConfig, formatError(error), 'info');
this.failedLocales.push({ uid, code });
} else {
log(this.importConfig, `Language '${code}' failed to import`, 'error');
log(this.importConfig, formatError(error), 'error');
this.failedLocales.push({ uid, code });
}
};
return await this.makeConcurrentCall({
processName: 'Import locales',
17 changes: 9 additions & 8 deletions packages/contentstack-import/src/types/index.ts
Original file line number Diff line number Diff line change
@@ -48,41 +48,42 @@ export type ModuleClassParams = {
moduleName: Modules;
};

export interface Extensions{
export interface Extensions {
dirName: string;
fileName: string;
validKeys: string[];
dependencies?: Modules[];
};
}

export interface MarketplaceAppsConfig{
export interface MarketplaceAppsConfig {
dirName: string;
fileName: string;
dependencies?: Modules[];
};
}

export interface EnvironmentConfig {
dirName: string;
fileName: string;
dependencies?: Modules[];
}

export interface LabelConfig{
export interface LabelConfig {
dirName: string;
fileName: string;
}

export interface WebhookConfig{
export interface WebhookConfig {
dirName: string;
fileName: string;
}

export interface WorkflowConfig{
export interface WorkflowConfig {
dirName: string;
fileName: string;
invalidKeys: string[];
}

export interface CustomRoleConfig{
export interface CustomRoleConfig {
dirName: string;
fileName: string;
customRolesLocalesFileName: string;
5 changes: 5 additions & 0 deletions packages/contentstack-utilities/src/fs-utility/core.ts
Original file line number Diff line number Diff line change
@@ -57,6 +57,7 @@ export default class FsUtility {
chunkFileSize,
indexFileName,
defaultInitContent,
useIndexer = false,
createDirIfNotExist = true,
} = options;
this.metaHandler = metaHandler;
@@ -71,6 +72,10 @@ export default class FsUtility {
this.pageInfo.hasNextPage = keys(this.indexFileContent).length > 0;
this.defaultInitContent = defaultInitContent || (this.fileExt === 'json' ? '{' : '');

if (useIndexer) {
this.writeIndexer = this.indexFileContent;
}

if (createDirIfNotExist) {
this.createFolderIfNotExist(this.basePath);
}
10 changes: 4 additions & 6 deletions packages/contentstack-utilities/src/fs-utility/types.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
type FileType = "json" | "txt";
type FileType = 'json' | 'txt';

type Chunk =
| Record<string, unknown>[]
| Record<string, unknown>
| Array<unknown>
| string;
type Chunk = Record<string, unknown>[] | Record<string, unknown> | Array<unknown> | string;

type PageInfo = {
after: number;
@@ -73,6 +69,8 @@ type FsConstructorOptions = {

keepMetadata?: boolean;

useIndexer?: boolean;

metaHandler?: (array: any) => any;
};

0 comments on commit 1b44c1f

Please sign in to comment.