Skip to content
This repository has been archived by the owner on Apr 13, 2023. It is now read-only.

Commit

Permalink
feat: add custom resource to update search mappings (#474)
Browse files Browse the repository at this point in the history
  • Loading branch information
carvantes authored Oct 11, 2021
1 parent c9a2c78 commit e941aa7
Show file tree
Hide file tree
Showing 6 changed files with 163 additions and 24 deletions.
10 changes: 9 additions & 1 deletion cloudformation/elasticsearch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,9 @@ Resources:
UpdatePolicy:
EnableVersionUpgrade: true
Properties:
EBSOptions: # Assuming ~100GB storage requirement for PROD; min storage requirement is ~290GB https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/sizing-domains.html
EBSOptions:
# Assuming ~100GB storage requirement for PROD; min storage requirement is ~290GB https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/sizing-domains.html
# If you change the size of the Elasticsearch Domain, consider also updating the NUMBER_OF_SHARDS on the updateSearchMappings resource
EBSEnabled: true
VolumeType: gp2
VolumeSize: !If [isDev, 10, 73]
Expand Down Expand Up @@ -144,3 +146,9 @@ Resources:
Resource:
Fn::Sub: arn:${AWS::Partition}:es:${AWS::Region}:${AWS::AccountId}:domain/*
- !Ref AWS::NoValue
UpdateSearchMappingsCustomResource:
DependsOn: ElasticSearchDomain
Type: AWS::CloudFormation::CustomResource
Properties:
ServiceToken: !GetAtt UpdateSearchMappingsLambdaFunction.Arn # serverless by convention capitalizes first letter and suffixes with "LambdaFunction"
RandomValue: ${sls:instanceId} # This forces the upload to happen on every deployment
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"fhir-works-on-aws-interface": "11.1.0",
"fhir-works-on-aws-persistence-ddb": "3.8.2",
"fhir-works-on-aws-routing": "6.1.2",
"fhir-works-on-aws-search-es": "3.5.2",
"fhir-works-on-aws-search-es": "3.6.0",
"serverless-http": "^2.3.1",
"yargs": "^16.2.0"
},
Expand Down
67 changes: 67 additions & 0 deletions serverless.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,18 @@ functions:
environment:
GLUE_SCRIPTS_BUCKET: !Ref GlueScriptsBucket

updateSearchMappings:
timeout: 300
memorySize: 512
runtime: nodejs14.x
description: 'Custom resource Lambda to update the search mappings'
role: UpdateSearchMappingsLambdaRole
handler: updateSearchMappings/index.handler
disableLogs: true # needed to avoid race condition error "Resource of type 'AWS::Logs::LogGroup' already exists" since the custom resource lambda invocation may create the log group before CFN does
environment:
ELASTICSEARCH_DOMAIN_ENDPOINT: !Join ['', ['https://', !GetAtt ElasticSearchDomain.DomainEndpoint]]
NUMBER_OF_SHARDS: !If [isDev, 1, 3] # 133 indices, one per resource types

stepFunctions:
stateMachines:
BulkExportStateMachine: ${file(bulkExport/state-machine-definition.yaml)}
Expand Down Expand Up @@ -598,6 +610,61 @@ resources:
Resource:
- !GetAtt DynamodbKMSKey.Arn
- !GetAtt ElasticSearchKMSKey.Arn
UpdateSearchMappingsLambdaRole:
Type: AWS::IAM::Role
Metadata:
cfn_nag:
rules_to_suppress:
- id: W11
reason: '* only applies to X-Ray statement which does not define a group or sampling-rule'
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: 'Allow'
Principal:
Service: 'lambda.amazonaws.com'
Action: 'sts:AssumeRole'
Policies:
- PolicyName: 'DdbToEsLambdaPolicy'
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- logs:CreateLogStream
- logs:CreateLogGroup
- logs:PutLogEvents
Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:*:*'
- Effect: Allow
Action:
- xray:PutTraceSegments
- xray:PutTelemetryRecords
Resource:
- '*'
- Effect: Allow
Action:
- 'es:ESHttpPost'
- 'es:ESHttpPut'
- 'es:ESHttpHead'
Resource:
- !Join ['', [!GetAtt ElasticSearchDomain.Arn, '/*']]
- PolicyName: 'KMSPolicy'
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- 'kms:Describe*'
- 'kms:Get*'
- 'kms:List*'
- 'kms:Encrypt'
- 'kms:Decrypt'
- 'kms:ReEncrypt*'
- 'kms:GenerateDataKey'
- 'kms:GenerateDataKeyWithoutPlaintext'
Resource:
- !GetAtt ElasticSearchKMSKey.Arn
DdbToEsDLQ:
Metadata:
cfn_nag:
Expand Down
2 changes: 1 addition & 1 deletion src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const { IS_OFFLINE, ENABLE_MULTI_TENANCY } = process.env;

const enableMultiTenancy = ENABLE_MULTI_TENANCY === 'true';

const fhirVersion: FhirVersion = '4.0.1';
export const fhirVersion: FhirVersion = '4.0.1';
const baseResources = fhirVersion === '4.0.1' ? BASE_R4_RESOURCES : BASE_STU3_RESOURCES;
const authService = IS_OFFLINE ? stubs.passThroughAuthz : new RBACHandler(RBACRules(baseResources), fhirVersion);
const dynamoDbDataService = new DynamoDbDataService(DynamoDb, false, { enableMultiTenancy });
Expand Down
78 changes: 78 additions & 0 deletions updateSearchMappings/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*
*/

import { getSearchMappings, SearchMappingsManager } from 'fhir-works-on-aws-search-es';
import axios from 'axios';
import { fhirVersion } from '../src/config';

const sendCfnResponse = async (event: any, status: 'SUCCESS' | 'FAILED', error?: Error) => {
if (error !== undefined) {
console.log(error);
}
const responseBody = JSON.stringify({
Status: status,
Reason: error?.message,
// The value of PhysicalResourceId doesn't really matter in this case.
// It just needs to be the same string on all responses to indicate that it is the same resource.
PhysicalResourceId: 'searchMappings',
StackId: event.StackId,
RequestId: event.RequestId,
LogicalResourceId: event.LogicalResourceId,
});
console.log(`Sending response to CFN: ${responseBody}`);
await axios.put(event.ResponseURL, responseBody);
};

/**
* Custom resource lambda handler that creates or updates the search mappings.
* Custom resource spec: See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-custom-resources.html
* @param event Custom resource request event. See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/crpg-ref-requests.html
*/
exports.handler = async (event: any) => {
console.log(event);
try {
if (process.env.ELASTICSEARCH_DOMAIN_ENDPOINT === undefined) {
throw new Error('Missing required env variable ELASTICSEARCH_DOMAIN_ENDPOINT');
}

if (process.env.NUMBER_OF_SHARDS === undefined) {
throw new Error('Missing required env variable NUMBER_OF_SHARDS');
}

const numberOfShards = Number.parseInt(process.env.NUMBER_OF_SHARDS, 10);
if (Number.isNaN(numberOfShards)) {
throw new Error('NUMBER_OF_SHARDS env variable is not a number');
}

const searchMappingsManager = new SearchMappingsManager({
numberOfShards,
searchMappings: getSearchMappings(fhirVersion),
ignoreMappingsErrorsForExistingIndices: true,
});

switch (event.RequestType as any) {
case 'Create':
case 'Update':
await searchMappingsManager.createOrUpdateMappings();
await sendCfnResponse(event, 'SUCCESS');
break;
case 'Delete':
console.log('Received Delete event. Doing nothing');
await sendCfnResponse(event, 'SUCCESS');
break;
default:
// This should never happen
await sendCfnResponse(
event,
'FAILED',
new Error(`Unknown event.RequestType value: ${event.RequestType}`),
);
break;
}
} catch (e) {
await sendCfnResponse(event, 'FAILED', e);
}
};
28 changes: 7 additions & 21 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -3075,21 +3075,6 @@ aws-sdk@^2.859.0:
uuid "3.3.2"
xml2js "0.4.19"

aws-sdk@^2.965.0:
version "2.970.0"
resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.970.0.tgz#dc258b61b4727dcb5130c494376b598eb19f827b"
integrity sha512-9+ktvE5xgpHr3RsFOcq1SrhXLvU+jUji44jbecFZb5C2lzoEEB29aeN39OLJMW0ZuOrR+3TNum8c3f8YVx6A7w==
dependencies:
buffer "4.9.2"
events "1.1.1"
ieee754 "1.1.13"
jmespath "0.15.0"
querystring "0.2.0"
sax "1.2.1"
url "0.10.3"
uuid "3.3.2"
xml2js "0.4.19"

aws-sign2@~0.7.0:
version "0.7.0"
resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
Expand Down Expand Up @@ -5896,16 +5881,17 @@ [email protected]:
serverless-http "^2.3.1"
uuid "^3.4.0"

fhir-works-on-aws-search-es@3.5.2:
version "3.5.2"
resolved "https://registry.yarnpkg.com/fhir-works-on-aws-search-es/-/fhir-works-on-aws-search-es-3.5.2.tgz#0005625198b64daf905371bc9c1c0c68430153ac"
integrity sha512-p94quGU8HbrVaVsft11zG73MSsgQrjmaTE7l99lqaaQouo7Ca0PQd9UMt7mUz+OQR1BQLZm0Fngqac+vHXnwfQ==
fhir-works-on-aws-search-es@3.6.0:
version "3.6.0"
resolved "https://registry.yarnpkg.com/fhir-works-on-aws-search-es/-/fhir-works-on-aws-search-es-3.6.0.tgz#4e37806f4ff19e82edf7112c470520d17f126c12"
integrity sha512-mMeT2dfcYMVsjILTggDGVX+roBlzZJ+aUHUATY2HomsQlViXSueF/ZJJZPQjqhU6I1YkVr7nRZIiPkq2C7kJug==
dependencies:
"@elastic/elasticsearch" "7.13"
aws-elasticsearch-connector "^8.2.0"
aws-sdk "^2.965.0"
aws-sdk "^2.1000.0"
date-fns "^2.19.0"
fhir-works-on-aws-interface "^10.0.0"
fhir-works-on-aws-interface "^11.1.0"
flat "^5.0.2"
lodash "^4.17.20"
nearley "^2.20.0"

Expand Down

0 comments on commit e941aa7

Please sign in to comment.