diff --git a/cloudformation/elasticsearch.yaml b/cloudformation/elasticsearch.yaml index 49fab3cb..4da7f7eb 100644 --- a/cloudformation/elasticsearch.yaml +++ b/cloudformation/elasticsearch.yaml @@ -93,7 +93,9 @@ Resources: UpdatePolicy: EnableVersionUpgrade: true Properties: - EBSOptions: # Assuming ~100GB storage requirement for PROD; min storage requirement is ~290GB https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/sizing-domains.html + EBSOptions: + # Assuming ~100GB storage requirement for PROD; min storage requirement is ~290GB https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/sizing-domains.html + # If you change the size of the Elasticsearch Domain, consider also updating the NUMBER_OF_SHARDS on the updateSearchMappings resource EBSEnabled: true VolumeType: gp2 VolumeSize: !If [isDev, 10, 73] @@ -144,3 +146,9 @@ Resources: Resource: Fn::Sub: arn:${AWS::Partition}:es:${AWS::Region}:${AWS::AccountId}:domain/* - !Ref AWS::NoValue + UpdateSearchMappingsCustomResource: + DependsOn: ElasticSearchDomain + Type: AWS::CloudFormation::CustomResource + Properties: + ServiceToken: !GetAtt UpdateSearchMappingsLambdaFunction.Arn # serverless by convention capitalizes first letter and suffixes with "LambdaFunction" + RandomValue: ${sls:instanceId} # This forces the upload to happen on every deployment diff --git a/package.json b/package.json index f99ebad9..3eabb1fb 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "fhir-works-on-aws-interface": "11.1.0", "fhir-works-on-aws-persistence-ddb": "3.8.2", "fhir-works-on-aws-routing": "6.1.2", - "fhir-works-on-aws-search-es": "3.5.2", + "fhir-works-on-aws-search-es": "3.6.0", "serverless-http": "^2.3.1", "yargs": "^16.2.0" }, diff --git a/serverless.yaml b/serverless.yaml index a89aa218..07430076 100644 --- a/serverless.yaml +++ b/serverless.yaml @@ -178,6 +178,18 @@ functions: environment: GLUE_SCRIPTS_BUCKET: !Ref GlueScriptsBucket + updateSearchMappings: + timeout: 300 + memorySize: 512 + runtime: nodejs14.x + description: 'Custom resource Lambda to update the search mappings' + role: UpdateSearchMappingsLambdaRole + handler: updateSearchMappings/index.handler + disableLogs: true # needed to avoid race condition error "Resource of type 'AWS::Logs::LogGroup' already exists" since the custom resource lambda invocation may create the log group before CFN does + environment: + ELASTICSEARCH_DOMAIN_ENDPOINT: !Join ['', ['https://', !GetAtt ElasticSearchDomain.DomainEndpoint]] + NUMBER_OF_SHARDS: !If [isDev, 1, 3] # 133 indices, one per resource types + stepFunctions: stateMachines: BulkExportStateMachine: ${file(bulkExport/state-machine-definition.yaml)} @@ -598,6 +610,61 @@ resources: Resource: - !GetAtt DynamodbKMSKey.Arn - !GetAtt ElasticSearchKMSKey.Arn + UpdateSearchMappingsLambdaRole: + Type: AWS::IAM::Role + Metadata: + cfn_nag: + rules_to_suppress: + - id: W11 + reason: '* only applies to X-Ray statement which does not define a group or sampling-rule' + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Principal: + Service: 'lambda.amazonaws.com' + Action: 'sts:AssumeRole' + Policies: + - PolicyName: 'DdbToEsLambdaPolicy' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogStream + - logs:CreateLogGroup + - logs:PutLogEvents + Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:*:*' + - Effect: Allow + Action: + - xray:PutTraceSegments + - xray:PutTelemetryRecords + Resource: + - '*' + - Effect: Allow + Action: + - 'es:ESHttpPost' + - 'es:ESHttpPut' + - 'es:ESHttpHead' + Resource: + - !Join ['', [!GetAtt ElasticSearchDomain.Arn, '/*']] + - PolicyName: 'KMSPolicy' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - 'kms:Describe*' + - 'kms:Get*' + - 'kms:List*' + - 'kms:Encrypt' + - 'kms:Decrypt' + - 'kms:ReEncrypt*' + - 'kms:GenerateDataKey' + - 'kms:GenerateDataKeyWithoutPlaintext' + Resource: + - !GetAtt ElasticSearchKMSKey.Arn DdbToEsDLQ: Metadata: cfn_nag: diff --git a/src/config.ts b/src/config.ts index 6f315583..f46a15d8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -29,7 +29,7 @@ const { IS_OFFLINE, ENABLE_MULTI_TENANCY } = process.env; const enableMultiTenancy = ENABLE_MULTI_TENANCY === 'true'; -const fhirVersion: FhirVersion = '4.0.1'; +export const fhirVersion: FhirVersion = '4.0.1'; const baseResources = fhirVersion === '4.0.1' ? BASE_R4_RESOURCES : BASE_STU3_RESOURCES; const authService = IS_OFFLINE ? stubs.passThroughAuthz : new RBACHandler(RBACRules(baseResources), fhirVersion); const dynamoDbDataService = new DynamoDbDataService(DynamoDb, false, { enableMultiTenancy }); diff --git a/updateSearchMappings/index.ts b/updateSearchMappings/index.ts new file mode 100644 index 00000000..18ca33d0 --- /dev/null +++ b/updateSearchMappings/index.ts @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + * + */ + +import { getSearchMappings, SearchMappingsManager } from 'fhir-works-on-aws-search-es'; +import axios from 'axios'; +import { fhirVersion } from '../src/config'; + +const sendCfnResponse = async (event: any, status: 'SUCCESS' | 'FAILED', error?: Error) => { + if (error !== undefined) { + console.log(error); + } + const responseBody = JSON.stringify({ + Status: status, + Reason: error?.message, + // The value of PhysicalResourceId doesn't really matter in this case. + // It just needs to be the same string on all responses to indicate that it is the same resource. + PhysicalResourceId: 'searchMappings', + StackId: event.StackId, + RequestId: event.RequestId, + LogicalResourceId: event.LogicalResourceId, + }); + console.log(`Sending response to CFN: ${responseBody}`); + await axios.put(event.ResponseURL, responseBody); +}; + +/** + * Custom resource lambda handler that creates or updates the search mappings. + * Custom resource spec: See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-custom-resources.html + * @param event Custom resource request event. See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/crpg-ref-requests.html + */ +exports.handler = async (event: any) => { + console.log(event); + try { + if (process.env.ELASTICSEARCH_DOMAIN_ENDPOINT === undefined) { + throw new Error('Missing required env variable ELASTICSEARCH_DOMAIN_ENDPOINT'); + } + + if (process.env.NUMBER_OF_SHARDS === undefined) { + throw new Error('Missing required env variable NUMBER_OF_SHARDS'); + } + + const numberOfShards = Number.parseInt(process.env.NUMBER_OF_SHARDS, 10); + if (Number.isNaN(numberOfShards)) { + throw new Error('NUMBER_OF_SHARDS env variable is not a number'); + } + + const searchMappingsManager = new SearchMappingsManager({ + numberOfShards, + searchMappings: getSearchMappings(fhirVersion), + ignoreMappingsErrorsForExistingIndices: true, + }); + + switch (event.RequestType as any) { + case 'Create': + case 'Update': + await searchMappingsManager.createOrUpdateMappings(); + await sendCfnResponse(event, 'SUCCESS'); + break; + case 'Delete': + console.log('Received Delete event. Doing nothing'); + await sendCfnResponse(event, 'SUCCESS'); + break; + default: + // This should never happen + await sendCfnResponse( + event, + 'FAILED', + new Error(`Unknown event.RequestType value: ${event.RequestType}`), + ); + break; + } + } catch (e) { + await sendCfnResponse(event, 'FAILED', e); + } +}; diff --git a/yarn.lock b/yarn.lock index 7a29b084..c4c03a17 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3075,21 +3075,6 @@ aws-sdk@^2.859.0: uuid "3.3.2" xml2js "0.4.19" -aws-sdk@^2.965.0: - version "2.970.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.970.0.tgz#dc258b61b4727dcb5130c494376b598eb19f827b" - integrity sha512-9+ktvE5xgpHr3RsFOcq1SrhXLvU+jUji44jbecFZb5C2lzoEEB29aeN39OLJMW0ZuOrR+3TNum8c3f8YVx6A7w== - dependencies: - buffer "4.9.2" - events "1.1.1" - ieee754 "1.1.13" - jmespath "0.15.0" - querystring "0.2.0" - sax "1.2.1" - url "0.10.3" - uuid "3.3.2" - xml2js "0.4.19" - aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -5896,16 +5881,17 @@ fhir-works-on-aws-routing@6.1.2: serverless-http "^2.3.1" uuid "^3.4.0" -fhir-works-on-aws-search-es@3.5.2: - version "3.5.2" - resolved "https://registry.yarnpkg.com/fhir-works-on-aws-search-es/-/fhir-works-on-aws-search-es-3.5.2.tgz#0005625198b64daf905371bc9c1c0c68430153ac" - integrity sha512-p94quGU8HbrVaVsft11zG73MSsgQrjmaTE7l99lqaaQouo7Ca0PQd9UMt7mUz+OQR1BQLZm0Fngqac+vHXnwfQ== +fhir-works-on-aws-search-es@3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/fhir-works-on-aws-search-es/-/fhir-works-on-aws-search-es-3.6.0.tgz#4e37806f4ff19e82edf7112c470520d17f126c12" + integrity sha512-mMeT2dfcYMVsjILTggDGVX+roBlzZJ+aUHUATY2HomsQlViXSueF/ZJJZPQjqhU6I1YkVr7nRZIiPkq2C7kJug== dependencies: "@elastic/elasticsearch" "7.13" aws-elasticsearch-connector "^8.2.0" - aws-sdk "^2.965.0" + aws-sdk "^2.1000.0" date-fns "^2.19.0" - fhir-works-on-aws-interface "^10.0.0" + fhir-works-on-aws-interface "^11.1.0" + flat "^5.0.2" lodash "^4.17.20" nearley "^2.20.0"