From e0b7f10bc35deda298375bcbae04feeab2f54157 Mon Sep 17 00:00:00 2001 From: smayda Date: Wed, 21 Apr 2021 22:42:53 -0400 Subject: [PATCH 1/2] fix: improve error logging for ddbToEs sync --- src/AWS.ts | 4 +- src/ddbToEs/ddbToEsHelper.ts | 85 +++++++++++++++++------------------- src/ddbToEs/index.ts | 17 +++++++- 3 files changed, 57 insertions(+), 49 deletions(-) diff --git a/src/AWS.ts b/src/AWS.ts index de6515d..d85b1c6 100644 --- a/src/AWS.ts +++ b/src/AWS.ts @@ -12,13 +12,13 @@ const AWSWithXray = AWSXRay.captureAWS(AWS); const { IS_OFFLINE } = process.env; if (IS_OFFLINE === 'true') { - AWS.config.update({ + AWSWithXray.config.update({ region: process.env.AWS_REGION || 'us-west-2', accessKeyId: process.env.ACCESS_KEY, secretAccessKey: process.env.SECRET_KEY, }); } else { - AWS.config.update({ + AWSWithXray.config.update({ customUserAgent: process.env.CUSTOM_USER_AGENT, }); } diff --git a/src/ddbToEs/ddbToEsHelper.ts b/src/ddbToEs/ddbToEsHelper.ts index 65adbb6..00e25d6 100644 --- a/src/ddbToEs/ddbToEsHelper.ts +++ b/src/ddbToEs/ddbToEsHelper.ts @@ -72,7 +72,8 @@ export default class DdbToEsHelper { await this.ElasticSearch.indices.create(params); } } catch (error) { - console.log('Failed to check if index exist or create index', error); + console.error(`Failed to check if index: ${indexName} exist or create index`); + throw error; } } @@ -140,64 +141,58 @@ export default class DdbToEsHelper { // eslint-disable-next-line class-methods-use-this async logAndExecutePromises(promiseParamAndIds: PromiseParamAndId[]) { - const upsertAvailablePromiseParamAndIds = promiseParamAndIds.filter(paramAndId => { - return paramAndId.type === 'upsert-AVAILABLE'; - }); - - const upsertDeletedPromiseParamAndIds = promiseParamAndIds.filter(paramAndId => { - return paramAndId.type === 'upsert-DELETED'; - }); - - const deletePromiseParamAndIds = promiseParamAndIds.filter(paramAndId => { - return paramAndId.type === 'delete'; - }); - - console.log( - `Operation: upsert-AVAILABLE on resource Ids `, - upsertAvailablePromiseParamAndIds.map(paramAndId => { - return paramAndId.id; - }), - ); - // We're using allSettled-shim because as of 7/21/2020 'serverless-plugin-typescript' does not support // Promise.allSettled. allSettled.shim(); - // We need to execute creation of a resource before execute deleting of a resource, - // because a resource can be created and deleted, but not deleted then restored to AVAILABLE - // @ts-ignore - await Promise.allSettled( - upsertAvailablePromiseParamAndIds.map(paramAndId => { - return this.ElasticSearch.update(paramAndId.promiseParam); - }), - ); + await this.executePromiseBlock('upsert-AVAILABLE', promiseParamAndIds); + await this.executePromiseBlock('upsert-DELETED', promiseParamAndIds); + await this.executePromiseBlock('delete', promiseParamAndIds); + } - console.log( - `Operation: upsert-DELETED on resource Ids `, - upsertDeletedPromiseParamAndIds.map(paramAndId => { - return paramAndId.id; - }), - ); + // eslint-disable-next-line class-methods-use-this + private async executePromiseBlock(type: PromiseType, promiseParamAndIds: PromiseParamAndId[]) { + const filteredPromiseParamAndIds = promiseParamAndIds.filter(paramAndId => { + return paramAndId.type === type; + }); - // @ts-ignore - await Promise.allSettled( - upsertDeletedPromiseParamAndIds.map(paramAndId => { - return this.ElasticSearch.update(paramAndId.promiseParam); - }), - ); + if (filteredPromiseParamAndIds.length === 0) { + return; + } console.log( - `Operation: delete on resource Ids `, - deletePromiseParamAndIds.map(paramAndId => { + `Starting operation "${type}" on resource Ids: `, + filteredPromiseParamAndIds.map(paramAndId => { return paramAndId.id; }), ); // @ts-ignore - await Promise.allSettled( - deletePromiseParamAndIds.map(paramAndId => { - return this.ElasticSearch.delete(paramAndId.promiseParam); + const results = await Promise.allSettled( + filteredPromiseParamAndIds.map(async paramAndId => { + try { + let response; + if (type === 'upsert-AVAILABLE' || type === 'upsert-DELETED') { + response = await this.ElasticSearch.update(paramAndId.promiseParam); + } else if (type === 'delete') { + response = await this.ElasticSearch.delete(paramAndId.promiseParam); + } else { + throw new Error(`unknown type: ${type}`); + } + return response; + } catch (e) { + console.error(`${type} failed on id: ${paramAndId.id}, due to error:\n${e}`); + throw e; + } }), ); + + // Throw rejected promises + const rejected = results + .filter((result: { status: string }) => result.status === 'rejected') + .map((result: { reason: string }) => result.reason); + if (rejected.length > 0) { + throw new Error(rejected); + } } } diff --git a/src/ddbToEs/index.ts b/src/ddbToEs/index.ts index 090fb96..843726a 100644 --- a/src/ddbToEs/index.ts +++ b/src/ddbToEs/index.ts @@ -25,7 +25,6 @@ export async function handleDdbToEsEvent(event: any) { const image = AWS.DynamoDB.Converter.unmarshall(ddbJsonImage); // Don't index binary files if (ddbToEsHelper.isBinaryResource(image)) { - console.log('This is a Binary resource. These are not searchable'); // eslint-disable-next-line no-continue continue; } @@ -47,6 +46,20 @@ export async function handleDdbToEsEvent(event: any) { await ddbToEsHelper.logAndExecutePromises(promiseParamAndIds); } catch (e) { - console.log('Failed to update ES records', e); + console.error( + 'Synchonization failed! The resources that could be effected are: ', + event.Records.map( + (record: { + eventName: string; + dynamodb: { OldImage: AWS.DynamoDB.AttributeMap; NewImage: AWS.DynamoDB.AttributeMap }; + }) => { + const image = record.eventName === REMOVE ? record.dynamodb.OldImage : record.dynamodb.NewImage; + return `{id: ${image.id.S}, vid: ${image.vid.N}}`; + }, + ), + ); + + console.error('Failed to update ES records', e); + throw e; } } From c41390ada7a6e495f738a6a1e325016787380e72 Mon Sep 17 00:00:00 2001 From: Robert Smayda Date: Fri, 23 Apr 2021 13:43:55 -0400 Subject: [PATCH 2/2] fix: Update src/ddbToEs/index.ts --- src/ddbToEs/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ddbToEs/index.ts b/src/ddbToEs/index.ts index 843726a..dbf6104 100644 --- a/src/ddbToEs/index.ts +++ b/src/ddbToEs/index.ts @@ -47,7 +47,7 @@ export async function handleDdbToEsEvent(event: any) { await ddbToEsHelper.logAndExecutePromises(promiseParamAndIds); } catch (e) { console.error( - 'Synchonization failed! The resources that could be effected are: ', + 'Synchronization failed! The resources that could be effected are: ', event.Records.map( (record: { eventName: string;