diff --git a/.github/workflows/dev-deployment.yml b/.github/workflows/dev-deployment.yml index e5d2fe18..6aa1d58f 100644 --- a/.github/workflows/dev-deployment.yml +++ b/.github/workflows/dev-deployment.yml @@ -1,8 +1,10 @@ -name: Find and Replace Dev Deployment +name: Push changes from Dev to Stage on: push: branches: [ dev ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: jobs: build: @@ -13,8 +15,8 @@ jobs: - name: Initiate - Dev uses: actions/checkout@v3 with: - token: ${{ secrets.PAT }} + token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.head_ref }} - name: Push changes - Init Stage - run: git push origin -f dev:stage \ No newline at end of file + run: git push origin -f dev:stage diff --git a/.github/workflows/stage-deployment.yml b/.github/workflows/stage-deployment.yml index 1e51a950..8205b640 100644 --- a/.github/workflows/stage-deployment.yml +++ b/.github/workflows/stage-deployment.yml @@ -1,8 +1,10 @@ -name: Find and Replace Stage Deployment +name: Push changes from Stage to Prod on: push: branches: [ stage ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: jobs: build: @@ -13,8 +15,8 @@ jobs: - name: Initiate - Stage uses: actions/checkout@v3 with: - token: ${{ secrets.PAT }} + token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.head_ref }} - name: Push changes - Init Prod - run: git push origin -f stage:prod \ No newline at end of file + run: git push origin -f stage:prod diff --git a/docs/cloudformation/app-geo-ca-stack.yml b/docs/cloudformation/app-geo-ca-stack.yml index 31134af5..04f97b7d 100644 --- a/docs/cloudformation/app-geo-ca-stack.yml +++ b/docs/cloudformation/app-geo-ca-stack.yml @@ -9,7 +9,7 @@ Parameters: SslCertArn: Type: AWS::SSM::Parameter::Value Default: /webpresence/app-geo-ca/ssl-cert-arn - Description: SSM parameter name for app.geo.ca ACM SSL Cert ARN + Description: SSM parameter name for app.geo.ca ACM SSL Cert ARN WebAclArn: Type: String Description: ARN of the WAF web ACL to use for CloudFront @@ -100,7 +100,7 @@ Resources: - 'logs:CreateLogGroup' - 'logs:CreateLogStream' - 'logs:PutLogEvents' - Resource: '*' + Resource: '*' AppCodeBuildProject: @@ -313,7 +313,7 @@ Resources: - Name: AppBuild RunOrder: 1 Configuration: - ProjectName: !Ref AppCodeBuildProject + ProjectName: !Ref AppCodeBuildProject - Name: Deploy Actions: - Name: Deploy @@ -371,6 +371,9 @@ Resources: LambdaFunctionAssociations: - EventType: origin-response LambdaFunctionARN: !Ref SecHeadersLambdaEdgeArn + FunctionAssociations: + - EventType: viewer-request + FunctionARN: !GetAtt CloudfrontAppHandler.FunctionMetadata.FunctionARN DefaultRootObject: index.html HttpVersion: http2 WebACLId: !Ref WebAclArn @@ -398,4 +401,35 @@ Resources: ResponsePagePath: /index.html ResponseCode: 200 - + CloudfrontAppHandler: + Type: AWS::CloudFront::Function + Properties: + AutoPublish: true + FunctionCode: > + function handler(event) { + var request = event.request; + // Redirects requests to the www. url to the non www. url. + if (request.headers && request.headers.host && request.headers.host.value.includes('www.')) { + var newurl = request.headers.host.value.replace('www.', '') + var qs = '' + Object.keys(request.querystring).forEach((e) => { + if (qs === '') { + qs += '?'; + } else { qs += '&' }; + qs += e + '=' + request.querystring[e].value; + }) + var response = { + statusCode: 301, + statusDescription: 'Moved Permanently', + headers: + { "location": { "value": 'https://' + newurl + request.uri + qs } } + } + return response; + } + + return request; + } + FunctionConfig: + Comment: A function run on every request to app.geo.ca + Runtime: cloudfront-js-1.0 + Name: app-handler diff --git a/docs/cloudformation/common-stack.yml b/docs/cloudformation/common-stack.yml index 1098b57d..a0157ade 100644 --- a/docs/cloudformation/common-stack.yml +++ b/docs/cloudformation/common-stack.yml @@ -12,8 +12,12 @@ Metadata: - GeocoreSslCertArn - GeocoreMetadataSslCertArn - GeocoreHarvestSslCertArn + - GeocorePygeoapiSslCertArn + - GeolocatorSslCertArn + - TiTilerSslCertArn - GeoCaSslCertArn - AppGeoCaSslCertArn + - SearchSslCertArn - GithubUsername - GithubKey @@ -26,6 +30,13 @@ Parameters: - prod Default: dev Description: Type of App Environment - dev stage prod + DynamoDBCreationBool: + Type: String + AllowedValues: + - true + - false + Default: false + Description: True or false to recreate all dynamodb tables DeploymentBucket: Description: S3 bucket where all the deployment related files are Type: String @@ -38,13 +49,25 @@ Parameters: Type: String GeocoreHarvestSslCertArn: Description: ARN of the SSL cert stored in ACM to be used for Geocore HNAP JSON Harvester - Type: String + Type: String + GeocorePygeoapiSslCertArn: + Description: ARN of the SSL cert stored in ACM to be used for Geocore pygeoapi + Type: String + GeolocatorSslCertArn: + Description: ARN of the SSL cert stored in ACM to be used for Geolocator + Type: String + TiTilerSslCertArn: + Description: ARN of the SSL cert stored in ACM to be used for TiTiler + Type: String GeoCaSslCertArn: Description: ARN of the SSL cert stored in ACM to be used for geo.ca Type: String AppGeoCaSslCertArn: Description: ARN of the SSL cert stored in ACM to be used for app.geo.ca Type: String + SearchSslCertArn: + Description: ARN of the SSL cert stored in ACM to be used for search-recherche.geocore.api.geo.ca + Type: String GithubUsername: Description: Github Username for accessing Canadian Geospatial Platform Type: String @@ -52,6 +75,10 @@ Parameters: Description: Github Key for accessing Canadian Geospatial Platform Type: String +Conditions: + IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] Resources: @@ -145,6 +172,14 @@ Resources: Type: String Value: !Ref Environment + DynamoDBCreationParam: + Type: AWS::SSM::Parameter + Properties: + Description: True or false to recreate all dynamodb tables + Name: /webpresence/dynamodb_creation + Type: String + Value: !Ref DynamoDBCreationBool + GeocoreHarvestSslCertArnParam: Type: AWS::SSM::Parameter Properties: @@ -152,6 +187,31 @@ Resources: Name: /webpresence/geocore-hnap-harvest/ssl-cert-arn Type: String Value: !Ref GeocoreHarvestSslCertArn + + GeocorePygeoapiSslCertArnParam: + Condition: IsDev + Type: AWS::SSM::Parameter + Properties: + Description: ARN of the SSL cert stored in ACM to be used for Geocore pygeoapi + Name: /webpresence/geocore-pygeoapi/ssl-cert-arn + Type: String + Value: !Ref GeocorePygeoapiSslCertArn + + GeolocatorSslCertArnParam: + Type: AWS::SSM::Parameter + Properties: + Description: ARN of the SSL cert stored in ACM to be used for Geolocator + Name: /webpresence/geolocator/ssl-cert-arn + Type: String + Value: !Ref GeolocatorSslCertArn + + TiTilerSslCertArnParam: + Type: AWS::SSM::Parameter + Properties: + Description: ARN of the SSL cert stored in ACM to be used for TiTiler + Name: /webpresence/titiler/ssl-cert-arn + Type: String + Value: !Ref TiTilerSslCertArn GeocoreSslCertArnParam: Type: AWS::SSM::Parameter @@ -185,6 +245,14 @@ Resources: Type: String Value: !Ref AppGeoCaSslCertArn + SearchSslCertArnParam: + Type: AWS::SSM::Parameter + Properties: + Description: ARN of the SSL cert stored in ACM to be used for search-recherche.geocore.api.geo.ca + Name: /webpresence/search-geocore-api-geo-ca/ssl-cert-arn + Type: String + Value: !Ref SearchSslCertArn + GithubUsernameParam: Type: AWS::SecretsManager::Secret Properties: diff --git a/docs/cloudformation/geo-ca-stack.yml b/docs/cloudformation/geo-ca-stack.yml index 7d753885..36160781 100644 --- a/docs/cloudformation/geo-ca-stack.yml +++ b/docs/cloudformation/geo-ca-stack.yml @@ -71,6 +71,18 @@ Resources: Action: s3:GetObject Resource: !Sub 'arn:aws:s3:::webpresence-geo-ca-${Environment}/*' + GeoCaResponseHeadersPolicy: + Type: AWS::CloudFront::ResponseHeadersPolicy + Properties: + ResponseHeadersPolicyConfig: + Name: geoca-response-headers-policy + Comment: Add custom response header to set cache-control + CustomHeadersConfig: + Items: + - Header: cache-control + Value: public, max-age=31536000 + Override: true + AppPipelineRole: Type: AWS::IAM::Role Properties: @@ -102,7 +114,7 @@ Resources: Resource: - Fn::Sub: arn:aws:lambda:* - + LambdaExecutionRole: Type: AWS::IAM::Role Properties: @@ -273,12 +285,17 @@ Resources: OriginAccessIdentity: !Join ['', ['origin-access-identity/cloudfront/', !Ref GeoCaOai]] Enabled: 'true' DefaultCacheBehavior: + Compress: true ViewerProtocolPolicy: redirect-to-https TargetOriginId: !Ref GeoCaStoreBucket CachePolicyId: 658327ea-f89d-4fab-a63d-7e88639e58f6 # https://docs.amazonaws.cn/en_us/AmazonCloudFront/latest/DeveloperGuide/using-managed-cache-policies.html + ResponseHeadersPolicyId: !Ref GeoCaResponseHeadersPolicy LambdaFunctionAssociations: - EventType: origin-response LambdaFunctionARN: !Ref SecHeadersLambdaEdgeArn + FunctionAssociations: + - EventType: viewer-request + FunctionARN: !GetAtt CloudfrontRootHandler.FunctionMetadata.FunctionARN DefaultRootObject: index.html HttpVersion: http2 WebACLId: !Ref WebAclArn @@ -305,3 +322,45 @@ Resources: ErrorCode: 404 ResponsePagePath: /404.html ResponseCode: 404 + + CloudfrontRootHandler: + Type: AWS::CloudFront::Function + Properties: + AutoPublish: true + FunctionCode: > + function handler(event) { + var request = event.request; + var uri = request.uri; + + // Check whether the URI is missing a file name. + if (uri.endsWith('/')) { + request.uri += 'index.html'; + } + // Check whether the URI is missing a file extension. + else if (!uri.includes('.')) { + request.uri += '/index.html'; + } + // Redirects requests to the www. url to the non www. url. + if (request.headers && request.headers.host && request.headers.host.value.includes('www.')) { + var newurl = request.headers.host.value.replace('www.', '') + var qs = '' + Object.keys(request.querystring).forEach((e) => { + if (qs === '') { + qs += '?'; + } else { qs += '&' }; + qs += e + '=' + request.querystring[e].value; + }) + var response = { + statusCode: 301, + statusDescription: 'Moved Permanently', + headers: + { "location": { "value": 'https://' + newurl + request.uri + qs } } + } + return response; + } + return request; + } + FunctionConfig: + Comment: A function run on every request to geo.ca + Runtime: cloudfront-js-1.0 + Name: root-handler diff --git a/docs/cloudformation/geocore-document-search.yml b/docs/cloudformation/geocore-document-search.yml new file mode 100644 index 00000000..89b4bc35 --- /dev/null +++ b/docs/cloudformation/geocore-document-search.yml @@ -0,0 +1,305 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: patterns-s3-docrepo + +Parameters: + Environment: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/environment + Description: SSM parameter name for environment + DeploymentBucket: + Type: String + Default: document-parser-code + MetadataBucket: + Type: String + Default: webpresence-geocore-json-to-geojson-${Environment} + DocumentsBucketName: + Type: String + Default: 's3-documentparser-documents-${Environment}' + StagingBucketName: + Type: String + Default: 's3-documentparser-staging-${Environment}' + QueuedBucketName: + Type: String + Default: 's3-documentparser-queued-${Environment}' + OSdomain: + Type: String + Default: 'https://search-search-m6agsr5kq5tlt7z6tbe7axm5gy.ca-central-1.es.amazonaws.com' + +Globals: + Function: + Timeout: 15 + Environment: + Variables: + language: en + +Resources: + DocumentsBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: + Ref: DocumentsBucketName + StagingBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: + Ref: StagingBucketName + QueuedBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: + Ref: QueuedBucketName + + MySqsQueue: + Type: AWS::SQS::Queue + queryOSFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: queryOSindex.zip + Handler: app.handler + Runtime: nodejs16.x + MemorySize: 512 + Timeout: 10 + ReservedConcurrentExecutions: 1 + Environment: + Variables: + domain: + Ref: OSdomain + Events: + HttpApiEvent: + Type: HttpApi + Properties: + Path: / + Method: GET + AddToOSFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: addToOSindex.zip + Handler: app.handler + Runtime: nodejs16.x + MemorySize: 512 + Timeout: 10 + ReservedConcurrentExecutions: 1 + Environment: + Variables: + domain: + Ref: OSdomain + Policies: + - S3ReadPolicy: + BucketName: + Ref: QueuedBucketName + - Statement: + - Effect: Allow + Resource: '*' + Action: + - comprehend:* + Events: + MySQSEvent: + Type: SQS + Properties: + Queue: + Fn::GetAtt: + - MySqsQueue + - Arn + BatchSize: 1 + + BatchingFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: batchingFunction.zip + Handler: app.handler + Runtime: nodejs16.x + MemorySize: 512 + Environment: + Variables: + OutputBucket: + Ref: QueuedBucketName + Policies: + - S3CrudPolicy: + BucketName: + Ref: QueuedBucketName + - S3ReadPolicy: + BucketName: + Ref: StagingBucketName + Events: + OriginalTextUpload: + Type: S3 + Properties: + Bucket: + Ref: StagingBucket + Events: s3:ObjectCreated:* + Filter: + S3Key: + Rules: + - Name: suffix + Value: .txt + AddToQueueFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: addToQueueFunction.zip + Handler: app.handler + Runtime: nodejs16.x + Timeout: 15 + MemorySize: 512 + Environment: + Variables: + SQSqueueName: + Ref: MySqsQueue + Policies: + - SQSSendMessagePolicy: + QueueName: + Fn::GetAtt: + - MySqsQueue + - QueueName + - S3ReadPolicy: + BucketName: + Ref: QueuedBucketName + Events: + FileUpload: + Type: S3 + Properties: + Bucket: + Ref: QueuedBucket + Events: s3:ObjectCreated:* + ProcessJPGFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: processJPG.zip + Handler: app.handler + Runtime: nodejs16.x + MemorySize: 512 + Environment: + Variables: + OutputBucket: + Ref: QueuedBucketName + MaxLabels: 5 + MinConfidence: 60 + Policies: + - S3CrudPolicy: + BucketName: + Ref: QueuedBucketName + - S3ReadPolicy: + BucketName: + Ref: DocumentsBucketName + - Statement: + - Effect: Allow + Resource: '*' + Action: + - rekognition:* + Events: + Upload: + Type: S3 + Properties: + Bucket: + Ref: DocumentsBucket + Events: s3:ObjectCreated:* + Filter: + S3Key: + Rules: + - Name: suffix + Value: .jpg + ProcessDOCXFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: processDOCX.zip + Handler: app.handler + Runtime: nodejs16.x + MemorySize: 512 + Environment: + Variables: + OutputBucket: + Ref: StagingBucketName + Policies: + - S3CrudPolicy: + BucketName: + Ref: StagingBucketName + - S3ReadPolicy: + BucketName: + Ref: DocumentsBucketName + Events: + Upload: + Type: S3 + Properties: + Bucket: + Ref: DocumentsBucket + Events: s3:ObjectCreated:* + Filter: + S3Key: + Rules: + - Name: suffix + Value: .docx + ProcessPDFFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: processPDF.zip + Handler: app.handler + Runtime: nodejs16.x + MemorySize: 512 + Environment: + Variables: + OutputBucket: + Ref: StagingBucketName + Policies: + - S3CrudPolicy: + BucketName: + Ref: StagingBucketName + - S3ReadPolicy: + BucketName: + Ref: DocumentsBucketName + Events: + Upload: + Type: S3 + Properties: + Bucket: + Ref: DocumentsBucket + Events: s3:ObjectCreated:* + Filter: + S3Key: + Rules: + - Name: suffix + Value: .pdf +Outputs: + DocumentsBucketName: + Description: Documents Bucket Name + Value: + Ref: DocumentsBucket + StagingBucketName: + Description: Staging Bucket Name + Value: + Ref: StagingBucket + QueuedBucketName: + Description: Queued Bucket Name + Value: + Ref: QueuedBucket + AddToOSFunctionARN: + Description: This AddToOSFunction function ARN. + Value: + Fn::GetAtt: + - AddToOSFunction + - Arn + AddToOSFunctionRoleARN: + Description: This ARN needs permission in the Opensearch configuration. + Value: + Fn::GetAtt: + - AddToOSFunctionRole + - Arn + queryOSFunctionARN: + Description: This ARN needs permission in the Opensearch configuration. + Value: + Fn::GetAtt: + - queryOSFunction + - Arn \ No newline at end of file diff --git a/docs/cloudformation/geocore-eo-rcm-ard-harvester.yml b/docs/cloudformation/geocore-eo-rcm-ard-harvester.yml new file mode 100644 index 00000000..d160b193 --- /dev/null +++ b/docs/cloudformation/geocore-eo-rcm-ard-harvester.yml @@ -0,0 +1,469 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: rcm-ard collection harvestor using serverless Scatter Gather pattern + + +Parameters: + RegionName: + Type: String + Default: 'ca-central-1' + Description: Default region for deployment. + SGRCMARDProcessesTableName: + Type: String + Default: eo_sg_rcm-ard_processes + Description: DynamoDB table name to store information by process triggered + SGRCMARDAggregateTableName: + Type: String + Default: eo_sg_rcm-ard_aggregate + Description: DynamoDB table name to store information by scatter_gatter aggregated + Environment: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/environment + Description: SSM parameter name for environment + DeploymentBucket: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/deployment-bucket + Description: S3 bucket where all deployment files are store + CreateProcessedBucket: + Type: String + Default: 'true' + Description: Parameter Store value to determine if ProcessedDataS3Bucket need to be created + CreateLinksBucket: + Type: String + Default: 'true' + Description: Parameter Store value to determine if EOItemLinksS3Bucket need to be created + +Conditions: + IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] + ShouldCreateProcessedBucket: !Equals [true, !Ref CreateProcessedBucket] + ShouldCreateLinksBucket: !Equals [true, !Ref CreateLinksBucket] + +Resources: + ProcessedDataS3Bucket: + Type: 'AWS::S3::Bucket' + Condition: ShouldCreateProcessedBucket + Properties: + BucketName: !Sub 'eo-sg-processed-data-rcm-ard-${Environment}' + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + AccessControl: Private + + EOItemLinksS3Bucket: + Type: 'AWS::S3::Bucket' + Condition: ShouldCreateLinksBucket + Properties: + BucketName: !Sub 'eo-sg-datacube-item-links-${Environment}' + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + AccessControl: Private + + EOSGLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + - arn:aws:iam::aws:policy/AmazonSQSFullAccess + - arn:aws:iam::aws:policy/AmazonS3FullAccess + +# ######################################################## +# # Lambda Function api links collector +# ######################################################## + CollectorFunction: + Type: AWS::Serverless::Function + Properties: + Handler: app.lambda_handler + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-collector-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-collector-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-collector-20240610-2100.zip + Runtime: python3.9 + MemorySize: 4096 + Timeout: 900 + Architectures: + - x86_64 + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Environment: + Variables: + GEOCORE_TEMPLATE_BUCKET_NAME: !Sub 'webpresence-geocore-template-${Environment}' + GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json' + ITEM_LINK_BUCKET_NAME: !Sub 'eo-sg-datacube-item-links-${Environment}' + PROCESSED_DATA_BUCKET_NAME: !Ref ProcessedDataS3Bucket + API_ROOT: 'https://www.eodms-sgdot.nrcan-rncan.gc.ca/stac' + ROOT_NAME: 'EODMS Datacube API / EODMS Cube de données API' + SOURCE: 'eodms' + SOURCESYSTEMNAME: 'ccmeo-eodms' + COLLECTION: 'rcm-ard' + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + CollectorFunctionRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'eo-sg-rcm-ard-collector-every-60mins-${Environment}' + Description: rcm-ard API links collector on every 60mins interval + State: DISABLED + ScheduleExpression: 'rate(60 minutes)' + Targets: + - Arn: !GetAtt [CollectorFunction, Arn] + Id: !Ref CollectorFunction + + + PermissionForEventsToInvokeCollectorLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref CollectorFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt CollectorFunctionRule.Arn + +######################################################## +# Lambda Function processor +######################################################## + ProcessorFunction: + Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction + Properties: + Handler: app.lambda_handler + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-processor-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-processor-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-processor-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + MemorySize: 4096 + Timeout: 900 + Runtime: python3.9 + Architectures: + - x86_64 + Environment: + Variables: + GEOCORE_TEMPLATE_BUCKET_NAME: !Sub 'webpresence-geocore-template-${Environment}' + GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json' + PROCESSED_DATA_BUCKET_NAME: !Ref ProcessedDataS3Bucket + API_ROOT: 'https://www.eodms-sgdot.nrcan-rncan.gc.ca/stac' + ROOT_NAME: 'EODMS Datacube API / EODMS Cube de données API' + SOURCE: 'eodms' + SOURCESYSTEMNAME: 'ccmeo-eodms' + COLLECTION: 'rcm-ard' + SG_PROCESSES_TABLE_NAME: !Ref SGRCMARDProcessesTableName + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + Events: + MySQSEvent: + Type: SQS + Properties: + Queue: !GetAtt ProcessorSQSQueue.Arn + BatchSize: 1 +######################################################## +# SQS to trigger the Lambda Function processor +######################################################## + ProcessorSQSQueue: + Type: AWS::SQS::Queue + Properties: + VisibilityTimeout: 901 + QueueName: "eo_processor_q_rcm-ard" + +######################################################## +# DynamoDB tables +######################################################## + SGProcessesDBTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: !Ref SGRCMARDProcessesTableName + KeySchema: + - AttributeName: scatter_gather_id + KeyType: HASH + - AttributeName: process_id + KeyType: RANGE + AttributeDefinitions: + - AttributeName: scatter_gather_id + AttributeType: S + - AttributeName: process_id + AttributeType: S + BillingMode: PAY_PER_REQUEST + StreamSpecification: + StreamViewType: NEW_IMAGE + + SGAggregateDBTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: !Ref SGRCMARDAggregateTableName + KeySchema: + - AttributeName: scatter_gather_id + KeyType: HASH + AttributeDefinitions: + - AttributeName: scatter_gather_id + AttributeType: S + BillingMode: PAY_PER_REQUEST + StreamSpecification: + StreamViewType: NEW_IMAGE + +######################################################## +# Lambda Function Scatter +######################################################## + ScatterFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-scatter-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-scatter-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-scatter-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Timeout: 900 + MemorySize: 4096 + Environment: + Variables: + ITEM_LINK_BUCKET_NAME: !Sub 'eo-sg-datacube-item-links-${Environment}' + JSON_FILENAME: 'rcm-ard-item-api.json' + SG_PROCESSES_TABLE_NAME: !Ref SGRCMARDProcessesTableName + SG_AGGREGATE_TABLE_NAME: !Ref SGRCMARDAggregateTableName + QUEUE_URL: !GetAtt ProcessorSQSQueue.QueueUrl + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + ScatterFunctionRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'eo-rcm-ard-sg-scatter-every-monday-${Environment}' + Description: scatter on once per week on Monday interval + State: DISABLED + ScheduleExpression: 'cron(0 8 ? * MON *)' + Targets: + - Arn: !GetAtt [ScatterFunction, Arn] + Id: !Ref ScatterFunction + + + PermissionForEventsToInvokeScatterLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref ScatterFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt ScatterFunctionRule.Arn + + +######################################################## +# Lambda Function Aggregator +######################################################## + AggregatorFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-aggregator-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-aggregator-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-aggregator-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Timeout: 900 + MemorySize: 4096 + Environment: + Variables: + SG_AGGREGATE_TABLE_NAME: !Ref SGRCMARDAggregateTableName + Layers: + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + EventSourceSGProcessesDBTableStream: + Type: AWS::Lambda::EventSourceMapping + Properties: + BatchSize: 100 + MaximumBatchingWindowInSeconds: 30 + Enabled: True + FilterCriteria: + Filters: + - Pattern: '{"eventName":["MODIFY"]}' + EventSourceArn: !GetAtt SGProcessesDBTable.StreamArn + FunctionName: !GetAtt AggregatorFunction.Arn + StartingPosition: LATEST + +######################################################## +# Lambda Function Gather +######################################################## + GatherFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-gather-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-gather-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-gather-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Timeout: 900 + MemorySize: 4096 + Environment: + Variables: + SG_AGGREGATE_TABLE_NAME: !Ref SGRCMARDAggregateTableName + Layers: + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + EventSourceSGAggregateDBTableStream: + Type: AWS::Lambda::EventSourceMapping + Properties: + BatchSize: 100 + MaximumBatchingWindowInSeconds: 30 + Enabled: True + FilterCriteria: + Filters: + - Pattern: '{"eventName":["MODIFY"]}' + EventSourceArn: !GetAtt SGAggregateDBTable.StreamArn + FunctionName: !GetAtt GatherFunction.Arn + StartingPosition: LATEST + + LogGroup: + Type: Custom::LogGroup + Properties: + ServiceToken: !ImportValue LogGroupHelperLambdaArn + LogGroupName: !Sub '/${Environment}/webpresence/eo_sg_rcm-ard_harvester' + RetentionInDays: 3653 + + +######################################################## +# Lambda Function Geocore to Parquet +######################################################## + GeocoretoParquetFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.9 + Role: !GetAtt ParquetLambdaExecutionRole.Arn + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + MemorySize: 10240 + Handler: app.lambda_handler + Timeout: 900 + Environment: + Variables: + GEOJSON_BUCKET_NAME: !Ref ProcessedDataS3Bucket + PARQUET_BUCKET_NAME: !Sub 'webpresence-geocore-geojson-to-parquet-${Environment}' + DYNAMODB_TABLE: 'analytics_popularity' + PARQUET_FILENAME: 'rcm-ard.parquet' + REGION_NAME: 'ca-central-1' + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + + ParquetLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + Policies: + - PolicyName: 'policy' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Action: + - 's3:*' + Resource: + - !Sub arn:aws:s3:::eo-sg-processed-data-rcm-ard-${Environment}/* + - !Sub arn:aws:s3:::eo-sg-processed-data-rcm-ard-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-hnap-json-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-hnap-json-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment} + + GeocoretoParquetFunctionRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'eo-sg-rcm-ard-parquet-every-tuesday-${Environment}' + Description: geocore to parquet on once per week on every Tuesday interval + State: DISABLED + ScheduleExpression: 'cron(0 8 ? * TUE *)' + Targets: + - Arn: !GetAtt [GeocoretoParquetFunction, Arn] + Id: !Ref GeocoretoParquetFunction + + + PermissionForEventsToInvokeGeocoretoParquetLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref GeocoretoParquetFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt GeocoretoParquetFunctionRule.Arn diff --git a/docs/cloudformation/geocore-eo-rcm-harvester.yml b/docs/cloudformation/geocore-eo-rcm-harvester.yml new file mode 100644 index 00000000..53b3c579 --- /dev/null +++ b/docs/cloudformation/geocore-eo-rcm-harvester.yml @@ -0,0 +1,471 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: rcm collection harvestor using serverless Scatter Gather pattern + + +Parameters: + RegionName: + Type: String + Default: 'ca-central-1' + Description: Default region for deployment. + SGrcmProcessesTableName: + Type: String + Default: eo_sg_rcm_processes + Description: DynamoDB table name to store information by process triggered + SGrcmAggregateTableName: + Type: String + Default: eo_sg_rcm_aggregate + Description: DynamoDB table name to store information by scatter_gatter aggregated + Environment: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/environment + Description: SSM parameter name for environment + DeploymentBucket: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/deployment-bucket + Description: S3 bucket where all deployment files are store + CreateProcessedBucket: + Type: String + Default: 'true' + Description: Parameter Store value to determine if ProcessedDataS3Bucket need to be created + CreateLinksBucket: + Type: String + Default: 'true' + Description: Parameter Store value to determine if EOItemLinksS3Bucket need to be created + +Conditions: + IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] + CreateProcessedBucket: !Equals [true, !Ref CreateProcessedBucket] + CreateLinksBucket: !Equals [true, !Ref CreateLinksBucket] + +Resources: + ProcessedDatarcmS3Bucket: + Type: 'AWS::S3::Bucket' + Condition: CreateProcessedBucket + Properties: + BucketName: !Sub 'eo-sg-processed-data-${Environment}' + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + AccessControl: Private + + EOItemLinksS3Bucket: + Type: 'AWS::S3::Bucket' + Condition: CreateLinksBucket + Properties: + BucketName: !Sub 'eo-sg-datacube-item-links-${Environment}' + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + AccessControl: Private + + EOSGLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + - arn:aws:iam::aws:policy/AmazonSQSFullAccess + - arn:aws:iam::aws:policy/AmazonS3FullAccess + +# ######################################################## +# # Lambda Function api links collector +# ######################################################## + CollectorFunction: + Type: AWS::Serverless::Function + Properties: + Handler: app.lambda_handler + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-collector-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-collector-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-collector-20240610-2100.zip + Runtime: python3.9 + MemorySize: 4096 + Timeout: 900 + Architectures: + - x86_64 + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Environment: + Variables: + GEOCORE_TEMPLATE_BUCKET_NAME: !Sub 'webpresence-geocore-template-${Environment}' + GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json' + ITEM_LINK_BUCKET_NAME: !Ref EOItemLinksS3Bucket + PROCESSED_DATA_BUCKET_NAME: !Ref ProcessedDataRCMS3Bucket + API_ROOT: 'https://www.eodms-sgdot.nrcan-rncan.gc.ca/stac' + ROOT_NAME: 'EODMS Datacube API / EODMS Cube de données API' + SOURCE: 'eodms' + SOURCESYSTEMNAME: 'ccmeo-eodms' + COLLECTION: 'rcm' + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + CollectorFunctionRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'eo-sg-rcm-collector-every-60mins-${Environment}' + Description: rcm API links collector on every 60mins interval + State: DISABLED + ScheduleExpression: 'rate(60 minutes)' + Targets: + - Arn: !GetAtt [CollectorFunction, Arn] + Id: !Ref CollectorFunction + + + PermissionForEventsToInvokeCollectorLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref CollectorFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt CollectorFunctionRule.Arn + + + +######################################################## +# Lambda Function processor +######################################################## + ProcessorFunction: + Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction + Properties: + Handler: app.lambda_handler + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-processor-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-processor-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-processor-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + MemorySize: 4096 + Timeout: 900 + Runtime: python3.9 + Architectures: + - x86_64 + Environment: + Variables: + GEOCORE_TEMPLATE_BUCKET_NAME: !Sub 'webpresence-geocore-template-${Environment}' + GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json' + PROCESSED_DATA_BUCKET_NAME: !Ref ProcessedDataRCMS3Bucket + API_ROOT: 'https://www.eodms-sgdot.nrcan-rncan.gc.ca/stac' + ROOT_NAME: 'EODMS Datacube API / EODMS Cube de données API' + SOURCE: 'eodms' + SOURCESYSTEMNAME: 'ccmeo-eodms' + COLLECTION: 'rcm' + SG_PROCESSES_TABLE_NAME: !Ref SGrcmProcessesTableName + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + Events: + MySQSEvent: + Type: SQS + Properties: + Queue: !GetAtt ProcessorSQSQueue.Arn + BatchSize: 1 +######################################################## +# SQS to trigger the Lambda Function processor +######################################################## + ProcessorSQSQueue: + Type: AWS::SQS::Queue + Properties: + VisibilityTimeout: 901 + QueueName: "eo_processor_q_rcm" + +######################################################## +# DynamoDB tables +######################################################## + SGProcessesDBTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: !Ref SGrcmProcessesTableName + KeySchema: + - AttributeName: scatter_gather_id + KeyType: HASH + - AttributeName: process_id + KeyType: RANGE + AttributeDefinitions: + - AttributeName: scatter_gather_id + AttributeType: S + - AttributeName: process_id + AttributeType: S + BillingMode: PAY_PER_REQUEST + StreamSpecification: + StreamViewType: NEW_IMAGE + + SGAggregateDBTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: !Ref SGrcmAggregateTableName + KeySchema: + - AttributeName: scatter_gather_id + KeyType: HASH + AttributeDefinitions: + - AttributeName: scatter_gather_id + AttributeType: S + BillingMode: PAY_PER_REQUEST + StreamSpecification: + StreamViewType: NEW_IMAGE + +######################################################## +# Lambda Function Scatter +######################################################## + ScatterFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-scatter-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-scatter-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-scatter-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Timeout: 900 + MemorySize: 4096 + Environment: + Variables: + ITEM_LINK_BUCKET_NAME: !Ref EOItemLinksS3Bucket + JSON_FILENAME: 'rcm-item-api.json' + SG_PROCESSES_TABLE_NAME: !Ref SGrcmProcessesTableName + SG_AGGREGATE_TABLE_NAME: !Ref SGrcmAggregateTableName + QUEUE_URL: !GetAtt ProcessorSQSQueue.QueueUrl + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + ScatterFunctionRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'eo-rcm-sg-scatter-every-monday-${Environment}' + Description: scatter on once per week on Monday interval + State: DISABLED + ScheduleExpression: 'cron(0 8 ? * MON *)' + Targets: + - Arn: !GetAtt [ScatterFunction, Arn] + Id: !Ref ScatterFunction + + + PermissionForEventsToInvokeScatterLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref ScatterFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt ScatterFunctionRule.Arn + + +######################################################## +# Lambda Function Aggregator +######################################################## + AggregatorFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-aggregator-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-aggregator-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-aggregator-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Timeout: 900 + MemorySize: 4096 + Environment: + Variables: + SG_AGGREGATE_TABLE_NAME: !Ref SGrcmAggregateTableName + Layers: + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + EventSourceSGProcessesDBTableStream: + Type: AWS::Lambda::EventSourceMapping + Properties: + BatchSize: 100 + MaximumBatchingWindowInSeconds: 30 + Enabled: True + FilterCriteria: + Filters: + - Pattern: '{"eventName":["MODIFY"]}' + EventSourceArn: !GetAtt SGProcessesDBTable.StreamArn + FunctionName: !GetAtt AggregatorFunction.Arn + StartingPosition: LATEST + +######################################################## +# Lambda Function Gather +######################################################## + GatherFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-gather-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-gather-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/rcm/rcm-sg-gather-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Timeout: 900 + MemorySize: 4096 + Environment: + Variables: + SG_AGGREGATE_TABLE_NAME: !Ref SGrcmAggregateTableName + Layers: + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + EventSourceSGAggregateDBTableStream: + Type: AWS::Lambda::EventSourceMapping + Properties: + BatchSize: 100 + MaximumBatchingWindowInSeconds: 30 + Enabled: True + FilterCriteria: + Filters: + - Pattern: '{"eventName":["MODIFY"]}' + EventSourceArn: !GetAtt SGAggregateDBTable.StreamArn + FunctionName: !GetAtt GatherFunction.Arn + StartingPosition: LATEST + + LogGroup: + Type: Custom::LogGroup + Properties: + ServiceToken: !ImportValue LogGroupHelperLambdaArn + LogGroupName: !Sub '/${Environment}/webpresence/eo_sg_rcm_harvester' + RetentionInDays: 3653 + + +######################################################## +# Lambda Function Geocore to Parquet +######################################################## + GeocoretoParquetFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.9 + Role: !GetAtt ParquetLambdaExecutionRole.Arn + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + MemorySize: 10240 + Handler: app.lambda_handler + Timeout: 900 + Environment: + Variables: + GEOJSON_BUCKET_NAME: !Ref ProcessedDataRCMS3Bucket + PARQUET_BUCKET_NAME: !Sub 'webpresence-geocore-geojson-to-parquet-${Environment}' + DYNAMODB_TABLE: 'analytics_popularity' + PARQUET_FILENAME: 'rcm.parquet' + REGION_NAME: 'ca-central-1' + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + + ParquetLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + Policies: + - PolicyName: 'policy' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Action: + - 's3:*' + Resource: + - !Sub arn:aws:s3:::eo-sg-processed-data-${Environment}/* + - !Sub arn:aws:s3:::eo-sg-processed-data-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-hnap-json-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-hnap-json-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment} + + GeocoretoParquetFunctionRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'eo-sg-rcm-parquet-every-tuesday-${Environment}' + Description: geocore to parquet on once per week on every Tuesday interval + State: DISABLED + ScheduleExpression: 'cron(0 8 ? * TUE *)' + Targets: + - Arn: !GetAtt [GeocoretoParquetFunction, Arn] + Id: !Ref GeocoretoParquetFunction + + + PermissionForEventsToInvokeGeocoretoParquetLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref GeocoretoParquetFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt GeocoretoParquetFunctionRule.Arn diff --git a/docs/cloudformation/geocore-eo-sentinel1-harvester.yml b/docs/cloudformation/geocore-eo-sentinel1-harvester.yml new file mode 100644 index 00000000..4fbe22a9 --- /dev/null +++ b/docs/cloudformation/geocore-eo-sentinel1-harvester.yml @@ -0,0 +1,472 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: Sentinel-1 collection harvestor using serverless Scatter Gather pattern + +Parameters: + RegionName: + Type: String + Default: 'ca-central-1' + Description: Default region for deployment. + SGSentinel1ProcessesTableName: + Type: String + Default: eo_sg_sentinel1_processes + Description: DynamoDB table name to store information by process triggered + SGSentinel1AggregateTableName: + Type: String + Default: eo_sg_sentinel1_aggregate + Description: DynamoDB table name to store information by scatter_gatter aggregated + Environment: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/environment + Description: SSM parameter name for environment + DeploymentBucket: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/deployment-bucket + Description: S3 bucket where all deployment files are store + CreateProcessedBucket: + Type: String + Default: 'true' + Description: Parameter Store value to determine if ProcessedDataS3Bucket need to be created + CreateLinksBucket: + Type: String + Default: 'true' + Description: Parameter Store value to determine if EOItemLinksS3Bucket need to be created + +Conditions: + IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] + ShouldCreateProcessedBucket: !Equals [true, !Ref CreateProcessedBucket] + ShouldCreateLinksBucket: !Equals [true, !Ref CreateLinksBucket] + +Resources: + ProcessedDataSentinel1S3Bucket: + Type: 'AWS::S3::Bucket' + Condition: ShouldCreateProcessedBucket + Properties: + BucketName: !Sub 'eo-sg-processed-data-sentinel1-${Environment}' + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + AccessControl: Private + + EOItemLinksS3Bucket: + Type: 'AWS::S3::Bucket' + Condition: ShouldCreateLinksBucket + Properties: + BucketName: !Sub 'eo-sg-datacube-item-links-${Environment}' + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + AccessControl: Private + + EOSGLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + - arn:aws:iam::aws:policy/AmazonSQSFullAccess + - arn:aws:iam::aws:policy/AmazonS3FullAccess + +# ######################################################## +# # Lambda Function api links collector +# ######################################################## + CollectorFunction: + Type: AWS::Serverless::Function + Properties: + Handler: app.lambda_handler + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-collector-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-collector-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-collector-20240610-2100.zip + Runtime: python3.9 + MemorySize: 4096 + Timeout: 900 + Architectures: + - x86_64 + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Environment: + Variables: + GEOCORE_TEMPLATE_BUCKET_NAME: !Sub 'webpresence-geocore-template-${Environment}' + GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json' + ITEM_LINK_BUCKET_NAME: !Ref EOItemLinksS3Bucket + PROCESSED_DATA_BUCKET_NAME: !Ref ProcessedDataSentinel1S3Bucket + API_ROOT: 'https://www.eodms-sgdot.nrcan-rncan.gc.ca/stac' + ROOT_NAME: 'EODMS Datacube API / EODMS Cube de données API' + SOURCE: 'eodms' + SOURCESYSTEMNAME: 'ccmeo-eodms' + COLLECTION: 'sentinel-1' + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + CollectorFunctionRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'eo-sg-sentinel1-collector-every-60mins-${Environment}' + Description: sentinel-1 API links collector on every 60mins interval + State: ENABLED + ScheduleExpression: 'rate(60 minutes)' + Targets: + - Arn: !GetAtt [CollectorFunction, Arn] + Id: !Ref CollectorFunction + + + PermissionForEventsToInvokeCollectorLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref CollectorFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt CollectorFunction.Arn + + + +######################################################## +# Lambda Function processor +######################################################## + ProcessorFunction: + Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction + Properties: + Handler: app.lambda_handler + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-processor-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-processor-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-processor-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + MemorySize: 4096 + Timeout: 900 + Runtime: python3.9 + Architectures: + - x86_64 + Environment: + Variables: + GEOCORE_TEMPLATE_BUCKET_NAME: !Sub 'webpresence-geocore-template-${Environment}' + GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json' + PROCESSED_DATA_BUCKET_NAME: !Ref ProcessedDataSentinel1S3Bucket + API_ROOT: 'https://www.eodms-sgdot.nrcan-rncan.gc.ca/stac' + ROOT_NAME: 'EODMS Datacube API / EODMS Cube de données API' + SOURCE: 'eodms' + SOURCESYSTEMNAME: 'ccmeo-eodms' + COLLECTION: 'sentinel-1' + SG_PROCESSES_TABLE_NAME: !Ref SGSentinel1ProcessesTableName + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + Events: + MySQSEvent: + Type: SQS + Properties: + Queue: !GetAtt ProcessorSQSQueue.Arn + BatchSize: 1 +######################################################## +# SQS to trigger the Lambda Function processor +######################################################## + ProcessorSQSQueue: + Type: AWS::SQS::Queue + Properties: + VisibilityTimeout: 901 + QueueName: "eo_processor_q_sentinel1" + +######################################################## +# DynamoDB tables +######################################################## + SGProcessesDBTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: !Ref SGSentinel1ProcessesTableName + KeySchema: + - AttributeName: scatter_gather_id + KeyType: HASH + - AttributeName: process_id + KeyType: RANGE + AttributeDefinitions: + - AttributeName: scatter_gather_id + AttributeType: S + - AttributeName: process_id + AttributeType: S + BillingMode: PAY_PER_REQUEST + StreamSpecification: + StreamViewType: NEW_IMAGE + + SGAggregateDBTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: !Ref SGSentinel1AggregateTableName + KeySchema: + - AttributeName: scatter_gather_id + KeyType: HASH + AttributeDefinitions: + - AttributeName: scatter_gather_id + AttributeType: S + BillingMode: PAY_PER_REQUEST + StreamSpecification: + StreamViewType: NEW_IMAGE + +######################################################## +# Lambda Function Scatter +######################################################## + ScatterFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-scatter-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-scatter-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-scatter-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Timeout: 900 + MemorySize: 4096 + Environment: + Variables: + ITEM_LINK_BUCKET_NAME: !Ref EOItemLinksS3Bucket + JSON_FILENAME: 'sentinel-1-item-api.json' + SG_PROCESSES_TABLE_NAME: !Ref SGSentinel1ProcessesTableName + SG_AGGREGATE_TABLE_NAME: !Ref SGSentinel1AggregateTableName + QUEUE_URL: !GetAtt ProcessorSQSQueue.QueueUrl + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + ScatterFunctionRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'eo-sg-scatter-every-monday-${Environment}' + Description: scatter on once per week on Monday interval + State: ENABLED + ScheduleExpression: 'cron(0 8 ? * MON *)' + Targets: + - Arn: !GetAtt [ScatterFunction, Arn] + Id: !Ref ScatterFunction + + + PermissionForEventsToInvokeScatterLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref ScatterFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt ScatterFunctionRule.Arn + + +######################################################## +# Lambda Function Aggregator +######################################################## + AggregatorFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-aggregator-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-aggregator-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-aggregator-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Timeout: 900 + MemorySize: 4096 + Environment: + Variables: + SG_AGGREGATE_TABLE_NAME: !Ref SGSentinel1AggregateTableName + Layers: + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + EventSourceSGProcessesDBTableStream: + Type: AWS::Lambda::EventSourceMapping + Properties: + BatchSize: 100 + MaximumBatchingWindowInSeconds: 30 + Enabled: True + FilterCriteria: + Filters: + - Pattern: '{"eventName":["MODIFY"]}' + EventSourceArn: !GetAtt SGProcessesDBTable.StreamArn + FunctionName: !GetAtt AggregatorFunction.Arn + StartingPosition: LATEST + +######################################################## +# Lambda Function Gather +######################################################## + GatherFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-gather-20240610-2100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-gather-20240610-2100.zip + - cloudformation-templates/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-gather-20240610-2100.zip + Role: !GetAtt EOSGLambdaExecutionRole.Arn + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + Timeout: 900 + MemorySize: 4096 + Environment: + Variables: + SG_AGGREGATE_TABLE_NAME: !Ref SGSentinel1AggregateTableName + Layers: + - arn:aws:lambda:ca-central-1:017000801446:layer:AWSLambdaPowertoolsPythonV2:46 + + EventSourceSGAggregateDBTableStream: + Type: AWS::Lambda::EventSourceMapping + Properties: + BatchSize: 100 + MaximumBatchingWindowInSeconds: 30 + Enabled: True + FilterCriteria: + Filters: + - Pattern: '{"eventName":["MODIFY"]}' + EventSourceArn: !GetAtt SGAggregateDBTable.StreamArn + FunctionName: !GetAtt GatherFunction.Arn + StartingPosition: LATEST + + LogGroup: + Type: Custom::LogGroup + Properties: + ServiceToken: !ImportValue LogGroupHelperLambdaArn + LogGroupName: !Sub '/${Environment}/webpresence/eo_sg_sentinel1_harvester' + RetentionInDays: 3653 + + +######################################################## +# Lambda Function Geocore to Parquet +######################################################## + GeocoretoParquetFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.9 + Role: !GetAtt ParquetLambdaExecutionRole.Arn + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + MemorySize: 10240 + Handler: app.lambda_handler + Timeout: 900 + Environment: + Variables: + GEOJSON_BUCKET_NAME: !Ref ProcessedDataSentinel1S3Bucket + PARQUET_BUCKET_NAME: !Sub 'webpresence-geocore-geojson-to-parquet-${Environment}' + DYNAMODB_TABLE: 'analytics_popularity' + PARQUET_FILENAME: 'sentinel1.parquet' + REGION_NAME: 'ca-central-1' + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + + ParquetLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + Policies: + - PolicyName: 'policy' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Action: + - 's3:*' + Resource: + - !Sub arn:aws:s3:::eo-sg-processed-data-sentinel1-${Environment}/* + - !Sub arn:aws:s3:::eo-sg-processed-data-sentinel1-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-hnap-json-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-hnap-json-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment} + + + + GeocoretoParquetFunctionRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'eo-sg-sentinel1-parquet-every-tuesday-${Environment}' + Description: geocore to parquet on once per week on every Tuesday interval + State: ENABLED + ScheduleExpression: 'cron(0 8 ? * TUE *)' + Targets: + - Arn: !GetAtt [GeocoretoParquetFunction, Arn] + Id: !Ref GeocoretoParquetFunction + + + PermissionForEventsToInvokeGeocoretoParquetLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref GeocoretoParquetFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt GeocoretoParquetFunctionRule.Arn diff --git a/docs/cloudformation/geocore-hnap-harvest.yml b/docs/cloudformation/geocore-hnap-harvest.yml index eb087750..ad4732c1 100644 --- a/docs/cloudformation/geocore-hnap-harvest.yml +++ b/docs/cloudformation/geocore-hnap-harvest.yml @@ -25,6 +25,8 @@ Parameters: Conditions: IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] Resources: GeocoreHnapJsonHarvestBucket: @@ -75,17 +77,28 @@ Resources: HnapJsonHarvester2: Type: AWS::Serverless::Function Properties: - Runtime: python3.8 + Runtime: python3.9 Role: !GetAtt LambdaExecutionRole.Arn CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest.zip - MemorySize: 1024 + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230823-1800.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230823-1800.zip + - cloudformation-templates/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230823-1800.zip + MemorySize: 3009 Handler: app.lambda_handler Timeout: 900 Environment: Variables: BUCKET_NAME: !Ref GeocoreHnapJsonHarvestBucket + GEOJSON_BUCKET_NAME: !Ref GeocoreHnapJsontoGeojsonBucket + BASE_URL: 'https://maps.canada.ca' + GN_JSON_RECORD_URL_START: 'https://maps.canada.ca/geonetwork/srv/api/0.1/records/' + RUN_INTERVAL_MINUTES: 11 Events: ApiEvent: Type: Api @@ -93,6 +106,33 @@ Resources: RestApiId: !Ref RestApi Path: /hnap_json_harvest Method: any + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:8 + + HnapJsonHarvesterRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'harvester-10mins-${Environment}' + Description: Harvester on a 10 minute interval + State: DISABLED + ScheduleExpression: 'rate(10 minutes)' + Targets: + - + Arn: + Fn::GetAtt: + - HnapJsonHarvester2 + - Arn + Id: !Ref HnapJsonHarvester2 + + PermissionForEventsToInvokeHarvesterLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref HnapJsonHarvester2 + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt + - HnapJsonHarvesterRule + - Arn GeocoreTransform: Type: AWS::Serverless::Function @@ -100,7 +140,14 @@ Resources: Runtime: nodejs14.x CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/geocore-transform/geocore-transform.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-transform/geocore-transform-20221114-1700.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-transform/geocore-transform-20221114-1700.zip + - cloudformation-templates/lambda/geocore-transform/geocore-transform-20221114-1700.zip Handler: index.handler MemorySize: 512 Timeout: 60 @@ -114,22 +161,57 @@ Resources: Properties: Bucket: !Ref GeocoreHnapJsonHarvestBucket Events: s3:ObjectCreated:* - + + GeocoretoParquetRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'parquet-60mins-${Environment}' + Description: GeocoretoParquet on a 60 minute interval + State: DISABLED + ScheduleExpression: 'rate(60 minutes)' + Targets: + - + Arn: + Fn::GetAtt: + - GeocoretoParquet + - Arn + Id: !Ref GeocoretoParquet + + PermissionForEventsToInvokeGeocoretoParquetLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref GeocoretoParquet + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt + - GeocoretoParquetRule + - Arn + GeocoretoParquet: Type: AWS::Serverless::Function Properties: - Runtime: python3.7 + Runtime: python3.9 Role: !GetAtt LambdaExecutionRole.Arn CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet.zip - MemorySize: 1024 + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + - cloudformation-templates/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip + MemorySize: 4096 Handler: app.lambda_handler Timeout: 900 Environment: Variables: GEOJSON_BUCKET_NAME: !Ref GeocoreHnapJsontoGeojsonBucket PARQUET_BUCKET_NAME: !Ref GeocoreGeojsontoParquetBucket + DYNAMODB_TABLE: 'analytics_popularity' + REGION_NAME: 'ca-central-1' + PARQUET_FILENAME: 'records.parquet' Events: ApiEvent: Type: Api @@ -137,7 +219,10 @@ Resources: RestApiId: !Ref RestApi Path: /geocore_to_parquet Method: any + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:12 + LambdaExecutionRole: Type: AWS::IAM::Role Properties: @@ -152,6 +237,7 @@ Resources: - sts:AssumeRole ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess Policies: - PolicyName: 'policy' PolicyDocument: @@ -167,7 +253,9 @@ Resources: - !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment} - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment}/* - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment} - + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment} + VerificationHeaderSecret: Type: AWS::SecretsManager::Secret Properties: @@ -356,4 +444,4 @@ Resources: Properties: ServiceToken: !ImportValue LogGroupHelperLambdaArn LogGroupName: !Sub '/${Environment}/webpresence/hnap_json_harvest' - RetentionInDays: 3653 + RetentionInDays: 3653 \ No newline at end of file diff --git a/docs/cloudformation/geocore-semantic-search-with-opensearch.yml b/docs/cloudformation/geocore-semantic-search-with-opensearch.yml new file mode 100644 index 00000000..aabd6d11 --- /dev/null +++ b/docs/cloudformation/geocore-semantic-search-with-opensearch.yml @@ -0,0 +1,495 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: 'AWS::Serverless-2016-10-31' +Description: Deploys semantic search with OpenSearch on geo.ca + +Parameters: + Environment: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/environment + Description: SSM parameter name for environment + DeploymentBucket: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/deployment-bucket + Description: S3 bucket where all deployment files are stored + SslCertArn: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/search-geocore-api-geo-ca/ssl-cert-arn + Description: SSM parameter name for search-recherche.geocore.api.geo.ca ACM SSL Cert ARN + WebAclArn: + Type: String + Description: ARN of the WAF web ACL to use for CloudFront + SecHeadersLambdaEdgeArn: + Type: String + Description: ARN of the Lambda@Edge function for injecting security headers to CloudFront + #Todo: change SagemakerEndpoint to a resource that can be referenced + SagemakerEndpoint: + Type: String + Default: semantic-search-model-1709665975 + Description: sagamaker endpoint for the pretrained/finetuned models + OSDomainName: + Type: String + Default: semantic-search + Description: OpenSearch domain name + OpenSearchUsername: + AllowedPattern: '^[a-zA-Z0-9]+$' + Default: admin + Description: User name for the account that will be added to the OpenSearch cluster. + MaxLength: '25' + MinLength: '5' + Type: String + OpenSearchPassword: + AllowedPattern: '(?=^.{8,32}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9])(?=.*[a-z])|(?=.*[^A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9]))^.*' + Description: Password for the account named above. Must be at least 8 characters containing letters, numbers and symbols + MaxLength: '32' + MinLength: '8' + NoEcho: 'true' + Type: String + Default: Semantic123! +#Todo: change OSEndpoint to a resource that can be referenced + OSEndpoint: + Type: String + Default: search-semantic-search-dfcizxxxuj62dusl5skmeu3czu.ca-central-1.es.amazonaws.com + Description: OpenSearch endpoint + OSSecretID: + Type: String + Default: dev/OpenSearch/SemanticSearch + Description: SSM parameter name for OpenSearch user name and password + + +Conditions: + IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] + + +Resources: +######################################################## +# Set up OpenSearch instance +######################################################## + OpenSearchServiceDomain: + Type: AWS::OpenSearchService::Domain + Properties: + AccessPolicies: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + AWS: '*' + Action: 'es:*' + Resource: !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/*/* + EngineVersion: 'OpenSearch_2.11' + DomainName: semantic-search + ClusterConfig: + #InstanceType: "r6g.2xlarge.search" + InstanceType: "t3.medium.search" + EBSOptions: + EBSEnabled: True + VolumeSize: 20 + VolumeType: 'gp3' + AdvancedSecurityOptions: + AnonymousAuthEnabled: False + Enabled: True + InternalUserDatabaseEnabled: True + MasterUserOptions: + MasterUserName: !Sub ${OpenSearchUsername} + MasterUserPassword: !Sub ${OpenSearchPassword} + NodeToNodeEncryptionOptions: + Enabled: True + EncryptionAtRestOptions: + Enabled: True + KmsKeyId: alias/aws/es + DomainEndpointOptions: + EnforceHTTPS: True + + OpenSearchSecret: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub OpenSearchSecret-${AWS::StackName} + Description: OpenSearch username and password + SecretString: !Sub '{ "username" : "${OpenSearchUsername}", "password" : "${OpenSearchPassword}" }' + + +######################################################## +# Set up SageMaker notebook instance +######################################################## + CodeRepository: + Type: AWS::SageMaker::CodeRepository + Properties: + GitConfig: + RepositoryUrl: https://github.com/Canadian-Geospatial-Platform/semantic-search-with-amazon-opensearch + +#Create a SageMaker Notebook Instance + NotebookInstance: + Type: AWS::SageMaker::NotebookInstance + Properties: + NotebookInstanceName: semantic-search-nb + InstanceType: ml.t3.2xlarge + RoleArn: !GetAtt NBRole.Arn + DefaultCodeRepository: !GetAtt CodeRepository.CodeRepositoryName + +#Defines an IAM role with permissions to interact with OpenSearch + OpenSearchSagemakerRole: + Type: AWS::IAM::Role + Properties: + RoleName: opensearch-sagemaker-role + Policies: + - PolicyName: SageMakerAccess + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "sagemaker:InvokeEndpointAsync" + - "sagemaker:InvokeEndpoint" + Resource: + - "*" + - Effect: Allow + Action: + - "bedrock:*" + Resource: + - "*" + AssumeRolePolicyDocument: # allows the OpenSearch service to assume this role + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - opensearchservice.amazonaws.com + Action: + - 'sts:AssumeRole' + +#Create an IAM role specifically for the SageMaker Notebook +#S3, DynamoDB, Secrets Manager, and passing the opensearch-sagemaker-role + NBRole: + Type: AWS::IAM::Role + Properties: + Policies: + - PolicyName: CustomSagemakerNotebookAccess + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "es:ESHttp*" + Resource: + - !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/* + - Effect: Allow + Action: + - "s3:*" + - "dynamodb:*" + Resource: + - "*" + - Effect: Allow + Action: + - "secretsmanager:GetSecretValue" + Resource: + - !Sub arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:* + - Effect: Allow + Action: + - "iam:PassRole" + Resource: + - !Sub arn:aws:iam::${AWS::AccountId}:role/opensearch-sagemaker-role + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonSageMakerFullAccess + - arn:aws:iam::aws:policy/AWSCloudFormationReadOnlyAccess + - arn:aws:iam::aws:policy/TranslateReadOnly + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - sagemaker.amazonaws.com + Action: + - 'sts:AssumeRole' + +######################################################## +# Lambda Function: invoke sagemaker model endpoints +# and perfrom KNN search in the OpenSearch domain +######################################################## + InvokeSagemakerEndpointPretrain: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref DeploymentBucket + Key: cloudformation-templates/lambda/semantic-search/Invoke-sagemaker-pretrain-20240508-2200.zip + + MemorySize: 512 + Timeout: 900 + Handler: app.lambda_handler + Runtime: python3.9 + Architectures: + - x86_64 + EphemeralStorage: + Size: 512 + Environment: + Variables: + MY_AWS_REGION: ca-central-1 + OS_ENDPOINT: !Ref OSEndpoint + OS_SECRET_ID: !Ref OSSecretID + SAGEMAKER_ENDPOINT: !Ref SagemakerEndpoint + PackageType: Zip + Role: !GetAtt InvokeSagemakerLambdaExecutionRole.Arn + Events: + ApiEvent: + Type: Api + Properties: + RestApiId: !Ref RestApi + Path: /* + Method: any + + InvokeSagemakerLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: InvokeSagemakerLambdaExecutionPolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - sagemaker:InvokeEndpoint + Resource: !Sub "arn:aws:sagemaker:${AWS::Region}:${AWS::AccountId}:endpoint/${SagemakerEndpoint}" + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${OSSecretID}*" + - Effect: Allow + Action: es:* + Resource: !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OSDomainName}/*" + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess +######################################################## +# Verification header SSM secret value +######################################################## + VerificationHeaderSecret: + Type: AWS::SecretsManager::Secret + Properties: + Description: Verification header secret value used to identify request coming from CloudFront to API Gateway + Name: /webpresence/search/verification-header-secret + GenerateSecretString: + ExcludePunctuation: true + PasswordLength: 32 +######################################################## +# CloudFront Origin Request Policy +# Specify how CloudFront handle cookies, headers, and query strings from the origin +######################################################## + OriginRequestPolicy: + Type: AWS::CloudFront::OriginRequestPolicy + Properties: + OriginRequestPolicyConfig: + CookiesConfig: + CookieBehavior: none #ignore cookies + HeadersConfig: + HeaderBehavior: whitelist + Headers: ["Accept"] + Name: search-cloudfront-to-api-gateway + QueryStringsConfig: + QueryStringBehavior: all +######################################################## +# CloudFront Distribution +# It will be referenced by the two A name in the Route53 +######################################################## + SearchDistribution: + Type: AWS::CloudFront::Distribution + DependsOn: VerificationHeaderSecret + Properties: + DistributionConfig: + Enabled: 'true' + Origins: + - Id: ApiOrigin + DomainName: !Sub ${RestApi}.execute-api.${AWS::Region}.amazonaws.com + CustomOriginConfig: + HTTPSPort: 443 + OriginProtocolPolicy: https-only + OriginSSLProtocols: + - TLSv1.2 + OriginPath: /live + OriginCustomHeaders: + - HeaderName: x-origin-verify + HeaderValue: '{{resolve:secretsmanager:/webpresence/search/verification-header-secret}}' + DefaultCacheBehavior: + AllowedMethods: [GET, HEAD, OPTIONS, PUT, PATCH, POST, DELETE] + ViewerProtocolPolicy: redirect-to-https + TargetOriginId: ApiOrigin + CachePolicyId: 4135ea2d-6df8-44a3-9df3-4b5a84be39ad # https://docs.amazonaws.cn/en_us/AmazonCloudFront/latest/DeveloperGuide/using-managed-cache-policies.html + OriginRequestPolicyId: !GetAtt OriginRequestPolicy.Id + LambdaFunctionAssociations: + - EventType: origin-response + LambdaFunctionARN: !Ref SecHeadersLambdaEdgeArn + HttpVersion: http2 + WebACLId: !Ref WebAclArn + Logging: + Bucket: !Sub 'webpresence-cloudfront-access-logs-${Environment}.s3.amazonaws.com' + IncludeCookies: true + Prefix: search + Aliases: + - !If + - IsProd + - search-recherche.geocore.api.geo.ca + - !Sub 'search-recherche.geocore-${Environment}.api.geo.ca' + - !If + - IsProd + - www.search-recherche.geocore.api.geo.ca + - !Sub 'www.search-recherche.geocore-${Environment}.api.geo.ca' + PriceClass: PriceClass_All + ViewerCertificate: + AcmCertificateArn: !Ref SslCertArn + SslSupportMethod: sni-only + MinimumProtocolVersion: TLSv1.2_2021 +######################################################## +# Web Application Firewall Access Control List (WebACL) +# Protect against common web exploits +######################################################## + RestApiWebAcl: + Type: AWS::WAFv2::WebACL + DependsOn: VerificationHeaderSecret + Properties: + Name: webpresence-search-cloudfront + Scope: REGIONAL + Description: "Restrict access to CloudFront" + DefaultAction: + Block: {} + VisibilityConfig: + SampledRequestsEnabled: true + CloudWatchMetricsEnabled: true + MetricName: webpresence-search-cloudfront + Rules: + - Name: webpresence-search-cloudfront-header-verification + Priority: 0 + Action: + Allow: {} + VisibilityConfig: + SampledRequestsEnabled: true + CloudWatchMetricsEnabled: true + MetricName: webpresence-search-cloudfront-header-verification + Statement: + ByteMatchStatement: + FieldToMatch: + SingleHeader: + Name: x-origin-verify + PositionalConstraint: EXACTLY + SearchString: '{{resolve:secretsmanager:/webpresence/search/verification-header-secret}}' + TextTransformations: + - Priority: 0 + Type: NONE +######################################################## +# Assocaite WAF ACL and an API Gateway stage +######################################################## + RestApiWebAclAssociations: + Type: AWS::WAFv2::WebACLAssociation + DependsOn: + - RestApiliveStage + Properties: + ResourceArn: !Sub 'arn:aws:apigateway:${AWS::Region}::/restapis/${RestApi}/stages/live' + WebACLArn: !GetAtt RestApiWebAcl.Arn +######################################################## +# The API definitions used for search API and lambda invokation +######################################################## + RestApi: + Type: AWS::Serverless::Api + Properties: + Name: search-opensearch + AccessLogSetting: + DestinationArn: !GetAtt LogGroup.Arn + Format: ' $context.identity.sourceIp $context.identity.caller $context.identity.user $context.requestTime $context.httpMethod $context.resourcePath $context.status $context.protocol $context.responseLength$context.error.message' + EndpointConfiguration: + Type: REGIONAL + OpenApiVersion: '2.0' + StageName: live + DefinitionBody: + swagger: '2.0' + info: + title: 'search-opensearch' + schemes: + - 'https' + paths: + /search-opensearch: + get: + consumes: + - "application/json" + produces: + - "application/json" + responses: #Method response:https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-apigateway-method-methodresponse.html + "200": + description: "200 response" + # schema: + # $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + x-amazon-apigateway-integration: #Integration response:https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-specification-api-gateway-extensions.html + type: aws_proxy + httpMethod: POST + uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${InvokeSagemakerEndpointPretrain.Arn}/invocations' + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\r\n \"method\" : \"$input.params('method')\",\r\n\ + \ \"searchString\" : \"$input.params('searchString')\"}" + #passthroughBehavior: "when_no_templates" + #contentHandling: "CONVERT_TO_TEXT" + options: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + # schema: + # $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + Access-Control-Allow-Methods: + type: "string" + Access-Control-Allow-Headers: + type: "string" + x-amazon-apigateway-integration: + type: "mock" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Methods: "'GET,OPTIONS'" + method.response.header.Access-Control-Allow-Headers: "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'" + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\"statusCode\": 200}" + # passthroughBehavior: "when_no_match" + # definitions: + # Empty: + # type: "object" + # title: "Empty Schema" + x-amazon-apigateway-gateway-responses: + DEFAULT_4XX: + responseParameters: + gatewayresponse.header.Access-Control-Allow-Methods: "'GET,OPTIONS'" + gatewayresponse.header.Access-Control-Allow-Origin: "'*'" + gatewayresponse.header.Access-Control-Allow-Headers: "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'" + DEFAULT_5XX: + responseParameters: + gatewayresponse.header.Access-Control-Allow-Methods: "'GET,OPTIONS'" + gatewayresponse.header.Access-Control-Allow-Origin: "'*'" + gatewayresponse.header.Access-Control-Allow-Headers: "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'" + + + + LogGroup: + Type: Custom::LogGroup + Properties: + ServiceToken: !ImportValue LogGroupHelperLambdaArn + LogGroupName: !Sub '/${Environment}/webpresence/search' + RetentionInDays: 3653 diff --git a/docs/cloudformation/geocore-similarity-engine-stack.yml b/docs/cloudformation/geocore-similarity-engine-stack.yml new file mode 100644 index 00000000..8f03e7c6 --- /dev/null +++ b/docs/cloudformation/geocore-similarity-engine-stack.yml @@ -0,0 +1,178 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: 'AWS::Serverless-2016-10-31' +Description: Deploys similarity engine solution + +Parameters: + Environment: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/environment + Description: SSM parameter name for environment + DeploymentBucket: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/deployment-bucket + Description: S3 bucket where all deployment files are stored + SslCertArn: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/geocore-hnap-harvest/ssl-cert-arn + Description: SSM parameter name for geocore Harvest ACM SSL Cert ARN + WebAclArn: + Type: String + Description: ARN of the WAF web ACL to use for CloudFront + SecHeadersLambdaEdgeArn: + Type: String + Description: ARN of the Lambda@Edge function for injecting security headers to CloudFront + + +Conditions: + IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] + +Resources: + NlpDataPreprocessingBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub 'webpresence-nlp-data-preprocessing-${Environment}' + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + AccessControl: Private + + SimilarityEngineDataPreprocessFunction: + Type: AWS::Serverless::Function + Properties: + Role: !GetAtt LambdaExecutionRole.Arn + PackageType: Image + ImageUri: + Fn::If: + - IsProd + - !Sub '${AWS::AccountId}.dkr.ecr.ca-central-1.amazonaws.com/similarity-engine-data-preprocess-python39-lambda-prod:1' + - Fn::If: + - IsStage + - !Sub '${AWS::AccountId}.dkr.ecr.ca-central-1.amazonaws.com/similarity-engine-data-preprocess-python39-lambda-stage:latest' + - !Sub '${AWS::AccountId}.dkr.ecr.ca-central-1.amazonaws.com/similarity-engine-data-preprocess-python39-v1-lambda:latest' + MemorySize: 4096 + Timeout: 900 # This sets the timeout to 900 seconds, which is 15 minutes + Environment: + Variables: + BUCKET_NAME_NLP: !Ref NlpDataPreprocessingBucket + BUCKET_NAME: !Sub 'webpresence-geocore-geojson-to-parquet-${Environment}' + FILE_NAME: "records.parquet" + +# CloudWatch Events Rule to trigger the Lambda function once per day + SimilarityEngineDataPreprocessRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'similarity-engine-data-preprocess-1day-${Environment}' + Description: Similarity engine data preprocess on a 1 day interval + State: ENABLED + ScheduleExpression: 'rate(1 day)' + Targets: + - + Arn: + Fn::GetAtt: + - SimilarityEngineDataPreprocessFunction + - Arn + Id: !Ref SimilarityEngineDataPreprocessFunction + + PermissionForEventsToInvokeSimilarityEngineDataPreprocessFunction: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref SimilarityEngineDataPreprocessFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt + - SimilarityEngineDataPreprocessRule + - Arn + + SimilarityEngineWord2VecFunction: + Type: AWS::Serverless::Function + Properties: + Role: !GetAtt LambdaExecutionRole.Arn + MemorySize: 4096 + Timeout: 900 # This sets the timeout to 900 seconds, which is 15 minutes + PackageType: Image + ImageUri: + Fn::If: + - IsProd + - !Sub '${AWS::AccountId}.dkr.ecr.ca-central-1.amazonaws.com/similarity-engine-word2vec-python39-lambda-prod:1' + - Fn::If: + - IsStage + - !Sub '${AWS::AccountId}.dkr.ecr.ca-central-1.amazonaws.com/similarity-engine-word2vec-python39-lambda-stage:1' + - !Sub '${AWS::AccountId}.dkr.ecr.ca-central-1.amazonaws.com/similarity-engine-word2vec-python39-lambda:2' + Environment: + Variables: + BUCKET_NAME_NLP: !Ref NlpDataPreprocessingBucket + BUCKET_NAME: !Sub 'webpresence-geocore-geojson-to-parquet-${Environment}' + FILE_NAME: "Processed_records.parquet" + FILE_NAME_ORIGINAL: "records.parquet" + + + + SimilarityEngineWord2VecRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'similarity-engine-word2vec-model-1day-${Environment}' + Description: Similarity engine model word2vec on a 1 day interval + State: ENABLED + ScheduleExpression: 'rate(1 day)' + Targets: + - + Arn: + Fn::GetAtt: + - SimilarityEngineWord2VecFunction + - Arn + Id: !Ref SimilarityEngineWord2VecFunction + + PermissionForEventsToInvokeSimilarityEngineWord2VecFunction: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref SimilarityEngineWord2VecFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt + - SimilarityEngineWord2VecRule + - Arn + + LambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + - arn:aws:iam::aws:policy/AmazonS3FullAccess + Policies: + - PolicyName: 'policy' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Action: + - 's3:*' + Resource: + - !Sub arn:aws:s3:::webpresence-nlp-data-preprocessing-${Environment}/* + - !Sub arn:aws:s3:::webpresence-nlp-data-preprocessing-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-geojson-to-parquet-${Environment} + + LogGroup: + Type: Custom::LogGroup + Properties: + ServiceToken: !ImportValue LogGroupHelperLambdaArn + LogGroupName: !Sub '/${Environment}/webpresence/similarity_engine' + RetentionInDays: 3653 diff --git a/docs/cloudformation/geocore-stac-harvest-stack.yml b/docs/cloudformation/geocore-stac-harvest-stack.yml new file mode 100644 index 00000000..a351c8be --- /dev/null +++ b/docs/cloudformation/geocore-stac-harvest-stack.yml @@ -0,0 +1,127 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: 'AWS::Serverless-2016-10-31' +Description: Deploys STAC datacube harvester solution + STAC to geocore translation + +Parameters: + Environment: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/environment + Description: SSM parameter name for environment + DeploymentBucket: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/deployment-bucket + Description: S3 bucket where all deployment files are stored + + +Conditions: + IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] + +Resources: + GeocoreFormatTemplateBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub 'webpresence-geocore-template-${Environment}' + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + AccessControl: Private + + GeocoreStacHarvestAndTransformFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.9 + Role: !GetAtt LambdaExecutionRole.Arn + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/stac-to-geocore/stac-to-geocore-20240628-1430.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/stac-to-geocore/stac-to-geocore-20240628-1430.zip + - cloudformation-templates/lambda/stac-to-geocore/stac-to-geocore-20240628-1430.zip + MemorySize: 4096 + Handler: app.lambda_handler + Timeout: 900 + Environment: + Variables: + GEOCORE_TEMPLATE_BUCKET_NAME: !Ref GeocoreFormatTemplateBucket + GEOCORE_TEMPLATE_NAME: 'geocore-format-null-template.json' + GEOCORE_TO_PARQUET_BUCKET_NAME: !Sub 'webpresence-geocore-json-to-geojson-${Environment}' + STAC_API_ROOT: 'https://datacube.services.geo.ca/api' + ROOT_NAME: 'CCMEO Datacube API / CCCOT Cube de données API' + SOURCE: 'ccmeo' + SOURCESYSTEMNAME: 'ccmeo-datacube' + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:8 + + StacHarvesterRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub 'stac-harvester-1day-${Environment}' + Description: Stac Harvester on a 1 day interval + State: ENABLED + ScheduleExpression: 'rate(1 day)' + Targets: + - + Arn: + Fn::GetAtt: + - GeocoreStacHarvestAndTransformFunction + - Arn + Id: !Ref GeocoreStacHarvestAndTransformFunction + + + PermissionForEventsToInvokeStacHarvesterLambda: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref GeocoreStacHarvestAndTransformFunction + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt + - StacHarvesterRule + - Arn + + LambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + Policies: + - PolicyName: 'policy' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: 'Allow' + Action: + - 's3:*' + Resource: + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-template-${Environment} + - !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment}/* + - !Sub arn:aws:s3:::webpresence-geocore-json-to-geojson-${Environment} + + LogGroup: + Type: Custom::LogGroup + Properties: + ServiceToken: !ImportValue LogGroupHelperLambdaArn + LogGroupName: !Sub '/${Environment}/webpresence/stac_harvester' + RetentionInDays: 3653 \ No newline at end of file diff --git a/docs/cloudformation/geocore-stack.yml b/docs/cloudformation/geocore-stack.yml index a96f2b42..f835351e 100644 --- a/docs/cloudformation/geocore-stack.yml +++ b/docs/cloudformation/geocore-stack.yml @@ -7,6 +7,10 @@ Parameters: Type: AWS::SSM::Parameter::Value Default: /webpresence/environment Description: SSM parameter name for environment + CreateDynamoDb: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/dynamodb_creation + Description: Parameter Store value to determine if DynamoDB tables need to be created DeploymentBucket: Type: AWS::SSM::Parameter::Value Default: /webpresence/deployment-bucket @@ -21,9 +25,16 @@ Parameters: SecHeadersLambdaEdgeArn: Type: String Description: ARN of the Lambda@Edge function for injecting security headers to CloudFront + OSDomain: + Type: String + Description: Open Search Domain Endpoint + Conditions: IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] + CreateDynamoDb: !Equals [true, !Ref CreateDynamoDb] Resources: @@ -78,6 +89,7 @@ Resources: DynamoDb: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: tag BillingMode: PAY_PER_REQUEST @@ -100,6 +112,7 @@ Resources: DynamoDbFoundational: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: foundational BillingMode: PAY_PER_REQUEST @@ -122,6 +135,7 @@ Resources: DynamoDbTheme: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: theme BillingMode: PAY_PER_REQUEST @@ -144,6 +158,7 @@ Resources: DynamoDbAnalytics: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: analytics BillingMode: PAY_PER_REQUEST @@ -161,6 +176,7 @@ Resources: DynamoDbAnnouncements: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: announcements BillingMode: PAY_PER_REQUEST @@ -180,9 +196,10 @@ Resources: - AttributeName: datetime KeyType: RANGE - + DynamoDbCommunity: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: community BillingMode: PAY_PER_REQUEST @@ -199,6 +216,7 @@ Resources: DynamoDbCommunityannouncement: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: community_announcement BillingMode: PAY_PER_REQUEST @@ -221,6 +239,7 @@ Resources: DynamoDbCommunitydata: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: community_data BillingMode: PAY_PER_REQUEST @@ -243,6 +262,7 @@ Resources: DynamoDbCommunityresource: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: community_resource BillingMode: PAY_PER_REQUEST @@ -265,6 +285,7 @@ Resources: DynamoDbCommunityroles: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: community_roles BillingMode: PAY_PER_REQUEST @@ -287,6 +308,7 @@ Resources: DynamoDbSavedrecords: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: saved_records BillingMode: PAY_PER_REQUEST @@ -309,7 +331,8 @@ Resources: DynamoDbSavedsearches: Type: AWS::DynamoDB::Table - Properties: + Condition: CreateDynamoDb + Properties: TableName: saved_searches BillingMode: PAY_PER_REQUEST PointInTimeRecoverySpecification: @@ -331,6 +354,7 @@ Resources: DynamoDbUsers: Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb Properties: TableName: users BillingMode: PAY_PER_REQUEST @@ -345,6 +369,46 @@ Resources: AttributeName: userId KeyType: HASH + DynamoDbPopularity: + Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb + Properties: + TableName: analytics_popularity + BillingMode: PAY_PER_REQUEST + PointInTimeRecoverySpecification: + PointInTimeRecoveryEnabled: true + AttributeDefinitions: + - + AttributeName: uuid + AttributeType: S + - + AttributeName: popularity + AttributeType: N + KeySchema: + - + AttributeName: uuid + KeyType: HASH + - + AttributeName: popularity + KeyType: RANGE + + DynamoDbGeoviewConfigService: + Type: AWS::DynamoDB::Table + Condition: CreateDynamoDb + Properties: + TableName: geoview_config_service + BillingMode: PAY_PER_REQUEST + PointInTimeRecoverySpecification: + PointInTimeRecoveryEnabled: true + AttributeDefinitions: + - + AttributeName: uuid + AttributeType: S + KeySchema: + - + AttributeName: uuid + KeyType: HASH + GeoCoreDatabase: Type: AWS::Glue::Database Properties: @@ -357,7 +421,7 @@ Resources: Properties: Database: !Ref GeoCoreDatabase WorkGroup: !Ref GeoCoreAthenaWorkGroup - QueryString: + QueryString: !Sub "CREATE EXTERNAL TABLE `metadata`( `features_type` string, `features_geometry_type` string, @@ -422,7 +486,10 @@ Resources: `features_properties_distributor` string, `features_properties_options` string, `features_properties_plugins` string, - `features_popularity` string) + `features_properties_sourcesystemname` string, + `features_popularity` int, + `features_properties_eocollection` string, + `features_properties_eofilters` string) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' STORED AS INPUTFORMAT @@ -441,7 +508,18 @@ Resources: 'rawDataSize'='-1', 'totalSize'='6676041', 'transient_lastDdlTime'='1647981845', - 'typeOfData'='file') + 'typeOfData'='file'); + + -- To add a new column(s) update below and run each manually. This named query is for documentation purpose only. + + ALTER EXTERNAL TABLE `metadata` + ADD COLUMN `features_popularity` int; + + ALTER EXTERNAL TABLE `metadata` + ADD COLUMN `features_properties_eocollection` string; + + ALTER EXTERNAL TABLE `metadata` + ADD COLUMN `features_properties_eofilters` string; " GeoCoreAthenaWorkGroup: @@ -452,14 +530,122 @@ Resources: ResultConfiguration: OutputLocation: !Sub 's3://${GeoCoreAthenaOutputBucket}' +# LambdaProcessIndex: +# Type: AWS::Serverless::Function +# Properties: +# Runtime: python3.8 +# # AutoPublishAlias: live +# CodeUri: +# Bucket: !Ref DeploymentBucket +# Key: +# Fn::If: +# - IsProd +# - cloudformation-templates/lambda/search-repo/add-to-os.zip +# - Fn::If: +# - IsStage +# - cloudformation-templates/lambda/search-repo/add-to-os.zip +# - cloudformation-templates/lambda/search-repo/add-to-os.zip +# Handler: index.handler +# Role: !GetAtt LambdaExecutionRole.Arn +# MemorySize: 512 +# Timeout: 60 +# Environment: +# Variables: +# #ES_HOST: +# # !Ref OSdomain +# #ES_INDEX: +# # !Ref search +# API_KEY: +# !Ref RestApiKey +# API: +# !Ref RestApi +# Events: +# ApiEvent: +# Type: Api +# Properties: +# RestApiId: !Ref RestApi +# Path: /* +# Method: any + +# LambdaResourceIndex: +# Type: AWS::Serverless::Function +# Properties: +# Runtime: nodejs16.x +# # AutoPublishAlias: live +# CodeUri: +# Bucket: !Ref DeploymentBucket +# Key: +# Fn::If: +# - IsProd +# - cloudformation-templates/lambda/search-repo/resource-parser.zip +# - Fn::If: +# - IsStage +# - cloudformation-templates/lambda/search-repo/resource-parser.zip +# - cloudformation-templates/lambda/search-repo/resource-parser.zip +# Handler: index.handler +# Role: !GetAtt LambdaExecutionRole.Arn +# MemorySize: 512 +# Timeout: 60 +# Environment: +# Variables: +# bucket: !Sub 's3-documentparser-documents-${Environment}' +# Events: +# ApiEvent: +# Type: Api +# Properties: +# RestApiId: !Ref RestApi +# Path: /* +# Method: any + LambdaId: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.9 + # AutoPublishAlias: live + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/id/id-20240612-1600.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/id/id-20240612-1600.zip + - cloudformation-templates/lambda/id/id-20240612-1600.zip + Handler: app.lambda_handler + Role: !GetAtt LambdaExecutionRole.Arn + MemorySize: 3009 + Timeout: 60 + Environment: + Variables: + CACHE_EXPIRY_IN_DAYS: '2' + PARQUET_BUCKET_NAME: !Sub 's3://webpresence-geocore-geojson-to-parquet-${Environment}' + Events: + ApiEvent: + Type: Api + Properties: + RestApiId: !Ref RestApi + Path: /* + Method: any + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:8 + + LambdaIdV1: Type: AWS::Serverless::Function Properties: Runtime: nodejs16.x # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/id/id.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/id/idv1-20230926-1100.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/id/idv1-20230926-1100.zip + - cloudformation-templates/lambda/id/idv1-20230926-1100.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -471,6 +657,30 @@ Resources: RestApiId: !Ref RestApi Path: /* Method: any + + KeepLambdaIdWarmScheduleRule: + Type: AWS::Events::Rule + Properties: + Description: KeepLambdaIdWarm + ScheduleExpression: rate(2 minutes) + State: ENABLED + Targets: + - + Arn: + Fn::GetAtt: + - LambdaId + - Arn + Id: TargetFunctionV1 + + PermissionToInvokeLambdaId: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref LambdaId + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt + - KeepLambdaIdWarmScheduleRule + - Arn LambdaGeo: Type: AWS::Serverless::Function @@ -479,7 +689,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/geo/geo.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geo/geo-20240627-2300.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geo/geo-20240627-2300.zip + - cloudformation-templates/lambda/geo/geocore-LambdaGeo-20240315.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -499,7 +716,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/featured/featured.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/featured/featured-220323.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/featured/featured-220323.zip + - cloudformation-templates/lambda/featured/featured-220323.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -519,7 +743,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/foundation/foundation_layer.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/foundation/foundation_layer-220323.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/foundation/foundation_layer-220323.zip + - cloudformation-templates/lambda/foundation/foundation_layer-220323.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -539,7 +770,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/analytics/analytics.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/analytics/analytics-220323.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/analytics/analytics-220323.zip + - cloudformation-templates/lambda/analytics/analytics-220323.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -559,7 +797,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/get_analytics/get_analytics.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/get_analytics/get_analytics-220323.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/get_analytics/get_analytics-220323.zip + - cloudformation-templates/lambda/get_analytics/get_analytics-220323.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -579,7 +824,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/announcements/announcement_add.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/announcements/announcement_add-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/announcements/announcement_add-230201.zip + - cloudformation-templates/lambda/announcements/announcement_add-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -599,7 +851,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/announcements/announcement_delete.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/announcements/announcement_delete-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/announcements/announcement_delete-230201.zip + - cloudformation-templates/lambda/announcements/announcement_delete-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -619,7 +878,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/announcements/announcement_get.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/announcements/announcement_get-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/announcements/announcement_get-230201.zip + - cloudformation-templates/lambda/announcements/announcement_get-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -639,7 +905,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/saved_record/saved_record.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/saved_record/saved_record-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/saved_record/saved_record-230201.zip + - cloudformation-templates/lambda/saved_record/saved_record-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -659,7 +932,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/saved_record/get_saved_record.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/saved_record/get_saved_record-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/saved_record/get_saved_record-230201.zip + - cloudformation-templates/lambda/saved_record/get_saved_record-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -679,7 +959,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/saved_record/delete_saved_record.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/saved_record/delete_saved_record-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/saved_record/delete_saved_record-230201.zip + - cloudformation-templates/lambda/saved_record/delete_saved_record-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -699,7 +986,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/saved_search/saved_search.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/saved_search/saved_search-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/saved_search/saved_search-230201.zip + - cloudformation-templates/lambda/saved_search/saved_search-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -719,7 +1013,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/saved_search/get_saved_searches.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/saved_search/get_saved_searches-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/saved_search/get_saved_searches-230201.zip + - cloudformation-templates/lambda/saved_search/get_saved_searches-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -739,7 +1040,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/saved_search/delete_saved_search.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/saved_search/delete_saved_search-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/saved_search/delete_saved_search-230201.zip + - cloudformation-templates/lambda/saved_search/delete_saved_search-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -759,7 +1067,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_add_member.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_add_member-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_add_member-230201.zip + - cloudformation-templates/lambda/community/community_add_member-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -779,7 +1094,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_add.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_add-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_add-230201.zip + - cloudformation-templates/lambda/community/community_add-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -799,7 +1121,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_announcement_add.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_announcement_add-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_announcement_add-230201.zip + - cloudformation-templates/lambda/community/community_announcement_add-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -819,7 +1148,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_announcement_delete.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_announcement_delete-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_announcement_delete-230201.zip + - cloudformation-templates/lambda/community/community_announcement_delete-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -839,7 +1175,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_announcement_edit.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_announcement_edit-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_announcement_edit-230201.zip + - cloudformation-templates/lambda/community/community_announcement_edit-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -859,7 +1202,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_announcement_get.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_announcement_get-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_announcement_get-230201.zip + - cloudformation-templates/lambda/community/community_announcement_get-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -879,7 +1229,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_data_add.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_data_add-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_data_add-230201.zip + - cloudformation-templates/lambda/community/community_data_add-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -899,7 +1256,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_data_delete.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_data_delete-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_data_delete-230201.zip + - cloudformation-templates/lambda/community/community_data_delete-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -919,7 +1283,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_data_get.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_data_get-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_data_get-230201.zip + - cloudformation-templates/lambda/community/community_data_get-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -939,7 +1310,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_resource_add.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_resource_add-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_resource_add-230201.zip + - cloudformation-templates/lambda/community/community_resource_add-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -959,7 +1337,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_resource_delete.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_resource_delete-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_resource_delete-230201.zip + - cloudformation-templates/lambda/community/community_resource_delete-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -979,7 +1364,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_resource_edit.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_resource_edit-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_resource_edit-230201.zip + - cloudformation-templates/lambda/community/community_resource_edit-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -999,7 +1391,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_resource_get.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_resource_get-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_resource_get-230201.zip + - cloudformation-templates/lambda/community/community_resource_get-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1019,7 +1418,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_delete_member.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_delete_member-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_delete_member-230201.zip + - cloudformation-templates/lambda/community/community_delete_member-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1039,7 +1445,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_edit_member.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_edit_member-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_edit_member-230201.zip + - cloudformation-templates/lambda/community/community_edit_member-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1059,7 +1472,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_edit.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_edit-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_edit-230201.zip + - cloudformation-templates/lambda/community/community_edit-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1079,7 +1499,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_get_members.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_get_members-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_get_members-230201.zip + - cloudformation-templates/lambda/community/community_get_members-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1099,7 +1526,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/community_get.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/community_get-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/community_get-230201.zip + - cloudformation-templates/lambda/community/community_get-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1119,7 +1553,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/my_community_announcements.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/my_community_announcements-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/my_community_announcements-230201.zip + - cloudformation-templates/lambda/community/my_community_announcements-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1139,7 +1580,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/my_community_data.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/my_community_data-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/my_community_data-230201.zip + - cloudformation-templates/lambda/community/my_community_data-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1159,7 +1607,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/community/my_community_resources.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/community/my_community_resources-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/community/my_community_resources-230201.zip + - cloudformation-templates/lambda/community/my_community_resources-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1179,7 +1634,14 @@ Resources: # AutoPublishAlias: live CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/translate/translate.zip + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/translate/translate-230201.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/translate/translate-230201.zip + - cloudformation-templates/lambda/translate/translate-230201.zip Handler: index.handler Role: !GetAtt LambdaExecutionRole.Arn MemorySize: 512 @@ -1195,17 +1657,25 @@ Resources: LambdaCollections: Type: AWS::Serverless::Function Properties: - Runtime: python3.7 + Runtime: python3.9 Role: !GetAtt LambdaExecutionRole.Arn CodeUri: Bucket: !Ref DeploymentBucket - Key: cloudformation-templates/lambda/collections/collections.zip - MemorySize: 1024 + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/collections/collections-20240612-1600.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/collections/collections-20240612-1600.zip + - cloudformation-templates/lambda/collections/collections-20240612-1600.zip + MemorySize: 3009 Handler: app.lambda_handler Timeout: 60 Environment: Variables: PARQUET_BUCKET_NAME: !Sub 's3://webpresence-geocore-geojson-to-parquet-${Environment}' + MAX_CHILD_OR_SIBLING_LENGTH: 5000 Events: ApiEvent: Type: Api @@ -1213,6 +1683,93 @@ Resources: RestApiId: !Ref RestApi Path: /collections Method: any + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:8 + + KeepCollectionsWarmScheduleRule: + Type: AWS::Events::Rule + Properties: + Description: KeepCollectionsWarm + ScheduleExpression: rate(2 minutes) + State: ENABLED + Targets: + - + Arn: + Fn::GetAtt: + - LambdaCollections + - Arn + Id: TargetFunctionV1 + + PermissionToInvokeLambdaId: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref LambdaCollections + Action: 'lambda:InvokeFunction' + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt + - KeepCollectionsWarmScheduleRule + - Arn + + LambdaPopularity: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.8 + Role: !GetAtt LambdaExecutionRole.Arn + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/geocore-filters/popularity-20230330-0800.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/geocore-filters/popularity-20230330-0800.zip + - cloudformation-templates/lambda/geocore-filters/popularity-20230330-0800.zip + MemorySize: 4096 + Handler: lambda_function.lambda_handler + Timeout: 900 + Environment: + Variables: + DYNAMODB_TABLE: 'analytics_popularity' + GEOJSON_BUCKET_NAME: !Sub 'webpresence-geocore-json-to-geojson-${Environment}' + GEONETWORK_POPULARITY_PATH: '' + PARQUET_BUCKET_NAME: !Sub 's3://webpresence-geocore-geojson-to-parquet-${Environment}/' + REGION_NAME: 'ca-central-1' + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python38:4 + + LambdaViewerConfigService: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.9 + Role: !GetAtt LambdaExecutionRole.Arn + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/viewer-config-service/viewer-config-service-20240924-1200.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/viewer-config-service/viewer-config-service-20240924-1200.zip + - cloudformation-templates/lambda/viewer-config-service/viewer-config-service-20240924-1200.zip + MemorySize: 3009 + Handler: app.lambda_handler + Timeout: 15 + Environment: + Variables: + GCS_TABLE: 'geoview_config_service' + GEOCORE_ID_API: 'https://geocore.api.geo.ca/id/v2' + RCS_CONFIG_PATH: 'https://maps.canada.ca/geonetwork/srv/api/v2/docs/' + Events: + ApiEvent: + Type: Api + Properties: + RestApiId: !Ref RestApi + Path: /vcs + Method: any + Layers: + - arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python39:11 LambdaExecutionRole: Type: AWS::IAM::Role @@ -1388,7 +1945,7 @@ Resources: Format: ' $context.identity.sourceIp $context.identity.caller $context.identity.user $context.requestTime $context.httpMethod $context.resourcePath $context.status $context.protocol $context.responseLength$context.error.message' EndpointConfiguration: Type: REGIONAL - OpenApiVersion: "2.0" + OpenApiVersion: "2.0" StageName: live DefinitionBody: swagger: "2.0" @@ -2086,7 +2643,7 @@ Resources: Access-Control-Allow-Origin: type: "string" x-amazon-apigateway-integration: - uri: "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaAnnouncementsget}/invocations" + uri: !Sub "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaAnnouncementsget}/invocations" httpMethod: "POST" responses: default: @@ -2140,7 +2697,7 @@ Resources: Access-Control-Allow-Origin: type: "string" x-amazon-apigateway-integration: - uri: "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaAnnouncementsadd}/invocations" + uri: !Sub "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaAnnouncementsadd}/invocations" httpMethod: "POST" responses: default: @@ -2192,7 +2749,7 @@ Resources: Access-Control-Allow-Origin: type: "string" x-amazon-apigateway-integration: - uri: "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaAnnouncementsdelete}/invocations" + uri: !Sub "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaAnnouncementsdelete}/invocations" httpMethod: "POST" responses: default: @@ -2246,7 +2803,7 @@ Resources: Access-Control-Allow-Origin: type: "string" x-amazon-apigateway-integration: - uri: "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaCollections}/invocations" + uri: !Sub "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaCollections}/invocations" httpMethod: "POST" responses: default: @@ -2286,7 +2843,7 @@ Resources: requestTemplates: application/json: "{\"statusCode\": 200}" passthroughBehavior: "when_no_match" - type: "mock" + type: "mock" /community/add: post: produces: @@ -3581,7 +4138,12 @@ Resources: ,\r\n \"lang\" : \"$input.params('lang')\",\r\n \"theme\" : \"$input.params('theme')\"\ ,\r\n \"type\": \"$input.params('type')\",\r\n \"org\": \"$input.params('org')\"\ ,\r\n \"min\": \"$input.params('min')\",\r\n \"max\": \"$input.params('max')\"\ - ,\r\n \"foundational\": \"$input.params('foundational')\"\r\n}" + ,\r\n \"foundational\": \"$input.params('foundational')\" + ,\r\n \"sort\": \"$input.params('sort')\" + ,\r\n \"source_system\": \"$input.params('sourcesystemname')\" + ,\r\n \"eo_collection\": \"$input.params('eocollection')\" + ,\r\n \"polarization\": \"$input.params('polarization')\" + ,\r\n \"orbit_direction\": \"$input.params('orbit')\"}" passthroughBehavior: "when_no_templates" contentHandling: "CONVERT_TO_TEXT" options: @@ -3614,6 +4176,120 @@ Resources: application/json: "{\"statusCode\": 200}" passthroughBehavior: "when_no_match" /id: + get: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + x-amazon-apigateway-integration: + type: "aws" + httpMethod: "POST" + uri: !Sub "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaIdV1}/invocations" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\r\n \"lang\" : \"$input.params('lang')\",\r\n \ + \ \"id\" : \"$input.params('id')\"\r\n}" + passthroughBehavior: "when_no_templates" + contentHandling: "CONVERT_TO_TEXT" + options: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + Access-Control-Allow-Methods: + type: "string" + Access-Control-Allow-Headers: + type: "string" + x-amazon-apigateway-integration: + type: "mock" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Methods: "'GET,OPTIONS'" + method.response.header.Access-Control-Allow-Headers: "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'" + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\"statusCode\": 200}" + passthroughBehavior: "when_no_match" + /id/v1: + get: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + x-amazon-apigateway-integration: + type: "aws" + httpMethod: "POST" + uri: !Sub "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaIdV1}/invocations" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\r\n \"lang\" : \"$input.params('lang')\",\r\n \ + \ \"id\" : \"$input.params('id')\"\r\n}" + passthroughBehavior: "when_no_templates" + contentHandling: "CONVERT_TO_TEXT" + options: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + Access-Control-Allow-Methods: + type: "string" + Access-Control-Allow-Headers: + type: "string" + x-amazon-apigateway-integration: + type: "mock" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Methods: "'GET,OPTIONS'" + method.response.header.Access-Control-Allow-Headers: "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'" + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\"statusCode\": 200}" + passthroughBehavior: "when_no_match" + /id/v2: get: consumes: - "application/json" @@ -3670,6 +4346,95 @@ Resources: requestTemplates: application/json: "{\"statusCode\": 200}" passthroughBehavior: "when_no_match" + /vcs: + get: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + x-amazon-apigateway-integration: + type: "aws" + httpMethod: "POST" + uri: !Sub "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaViewerConfigService}/invocations" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\r\n \"method\" : \"GET\",\r\n \ + \ \"lang\" : \"$input.params('lang')\",\r\n \ + \ \"metadata\" : \"$input.params('metadata')\",\r\n \ + \ \"id\" : \"$input.params('id')\"\r\n}" + passthroughBehavior: "when_no_templates" + contentHandling: "CONVERT_TO_TEXT" + post: + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + security: + - api_key: [] + x-amazon-apigateway-integration: + type: "aws" + httpMethod: "POST" + uri: !Sub "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaViewerConfigService}/invocations" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: | + { + "method" : "POST", + "body" : "$util.base64Encode($input.body)" + } + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + options: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + Access-Control-Allow-Methods: + type: "string" + Access-Control-Allow-Headers: + type: "string" + x-amazon-apigateway-integration: + type: "mock" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Methods: "'GET,OPTIONS'" + method.response.header.Access-Control-Allow-Headers: "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'" + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\"statusCode\": 200}" + passthroughBehavior: "when_no_match" /saved_records/add: post: produces: @@ -4056,6 +4821,62 @@ Resources: application/json: "{\"statusCode\": 200}" passthroughBehavior: "when_no_match" type: "mock" + /resourceindex: + post: + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + security: + - api_key: [] + x-amazon-apigateway-integration: + uri: "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaResourceIndex}/invocations" + httpMethod: "POST" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Origin: "'*'" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + type: "aws" + options: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + Access-Control-Allow-Methods: + type: "string" + Access-Control-Allow-Headers: + type: "string" + x-amazon-apigateway-integration: + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Methods: "'OPTIONS,POST'" + method.response.header.Access-Control-Allow-Headers: "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'" + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\"statusCode\": 200}" + passthroughBehavior: "when_no_match" + type: "mock" + + securityDefinitions: api_key: type: "apiKey" diff --git a/docs/cloudformation/geolocator-web-presence.yml b/docs/cloudformation/geolocator-web-presence.yml new file mode 100644 index 00000000..35dc6ea3 --- /dev/null +++ b/docs/cloudformation/geolocator-web-presence.yml @@ -0,0 +1,226 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: 'AWS::Serverless-2016-10-31' +Description: Deploys geolocator for web presence + +Parameters: + Environment: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/environment + Description: SSM parameter name for environment + DeploymentBucket: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/deployment-bucket + Description: S3 bucket where all deployment files are stored + SslCertArn: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/geolocator/ssl-cert-arn + Description: SSM parameter name for geolocator ACM SSL Cert ARN + WebAclArn: + Type: String + Description: ARN of the WAF web ACL to use for CloudFront + SecHeadersLambdaEdgeArn: + Type: String + Description: ARN of the Lambda@Edge function for injecting security headers to CloudFront + +Conditions: + IsProd: !Equals [prod, !Ref Environment] + +Resources: + Geolocator: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.7 + Role: !GetAtt LambdaExecutionRole.Arn + CodeUri: + Bucket: !Ref DeploymentBucket + Key: cloudformation-templates/lambda/geolocator/geolocator-commit-15b0de5722c2362d49cdfdbdfd2cb9f97c94059e.zip + MemorySize: 3009 + Handler: index.handler + Timeout: 900 + Environment: + Variables: + S3_BUCKET_NAME: !Sub 'webpresence-geolocator-${Environment}' + Events: + ApiEvent: + Type: Api + Properties: + RestApiId: !Ref RestApi + Path: /* + Method: any + + LambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + - arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess + Policies: + - PolicyName: GeolocatorS3AccessPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:ListBucket + Resource: + - !Join ['', ['arn:aws:s3:::', !Ref geolocatorS3Bucket]] + - !Join ['', ['arn:aws:s3:::', !Ref geolocatorS3Bucket, '/*']] + + VerificationHeaderSecret: + Type: AWS::SecretsManager::Secret + Properties: + Description: Verification header secret value used to identify request coming from CloudFront to API Gateway + Name: /webpresence/geolocator/verification-header-secret + GenerateSecretString: + ExcludePunctuation: true + PasswordLength: 32 + + OriginRequestPolicy: + Type: AWS::CloudFront::OriginRequestPolicy + Properties: + OriginRequestPolicyConfig: + CookiesConfig: + CookieBehavior: none + HeadersConfig: + HeaderBehavior: whitelist + Headers: ["Accept"] + Name: geolocator-cloudfront-to-api-gateway + QueryStringsConfig: + QueryStringBehavior: all + + GeolocatorDistribution: + Type: AWS::CloudFront::Distribution + DependsOn: VerificationHeaderSecret + Properties: + DistributionConfig: + Enabled: 'true' + Origins: + - Id: ApiOrigin + DomainName: !Sub ${RestApi}.execute-api.${AWS::Region}.amazonaws.com + CustomOriginConfig: + HTTPSPort: 443 + OriginProtocolPolicy: https-only + OriginSSLProtocols: + - TLSv1.2 + OriginPath: /live + OriginCustomHeaders: + - HeaderName: x-origin-verify + HeaderValue: '{{resolve:secretsmanager:/webpresence/geolocator/verification-header-secret}}' + DefaultCacheBehavior: + AllowedMethods: [GET, HEAD, OPTIONS, PUT, PATCH, POST, DELETE] + ViewerProtocolPolicy: redirect-to-https + TargetOriginId: ApiOrigin + CachePolicyId: 4135ea2d-6df8-44a3-9df3-4b5a84be39ad # https://docs.amazonaws.cn/en_us/AmazonCloudFront/latest/DeveloperGuide/using-managed-cache-policies.html + OriginRequestPolicyId: !GetAtt OriginRequestPolicy.Id + LambdaFunctionAssociations: + - EventType: origin-response + LambdaFunctionARN: !Ref SecHeadersLambdaEdgeArn + HttpVersion: http2 + WebACLId: !Ref WebAclArn + Logging: + Bucket: !Sub 'webpresence-cloudfront-access-logs-${Environment}.s3.amazonaws.com' + IncludeCookies: true + Prefix: geolocator + Aliases: + - !If + - IsProd + - geolocator.api.geo.ca + - !Sub 'geolocator-${Environment}.api.geo.ca' + - !If + - IsProd + - www.geolocator.api.geo.ca + - !Sub 'www.geolocator-${Environment}.api.geo.ca' + PriceClass: PriceClass_All + ViewerCertificate: + AcmCertificateArn: !Ref SslCertArn + SslSupportMethod: sni-only + MinimumProtocolVersion: TLSv1.2_2021 + + RestApiWebAcl: + Type: AWS::WAFv2::WebACL + DependsOn: VerificationHeaderSecret + Properties: + Name: webpresence-geolocator-cloudfront + Scope: REGIONAL + Description: "Restrict access to CloudFront" + DefaultAction: + Block: {} + VisibilityConfig: + SampledRequestsEnabled: true + CloudWatchMetricsEnabled: true + MetricName: webpresence-geolocator-cloudfront + Rules: + - Name: webpresence-geolocator-cloudfront-header-verification + Priority: 0 + Action: + Allow: {} + VisibilityConfig: + SampledRequestsEnabled: true + CloudWatchMetricsEnabled: true + MetricName: webpresence-geolocator-cloudfront-header-verification + Statement: + ByteMatchStatement: + FieldToMatch: + SingleHeader: + Name: x-origin-verify + PositionalConstraint: EXACTLY + SearchString: '{{resolve:secretsmanager:/webpresence/geolocator/verification-header-secret}}' + TextTransformations: + - Priority: 0 + Type: NONE + + RestApiWebAclAssociations: + Type: AWS::WAFv2::WebACLAssociation + DependsOn: + - RestApiliveStage + Properties: + ResourceArn: !Sub 'arn:aws:apigateway:${AWS::Region}::/restapis/${RestApi}/stages/live' + WebACLArn: !GetAtt RestApiWebAcl.Arn + + RestApi: + Type: AWS::Serverless::Api + Properties: + Name: geolocator + AccessLogSetting: + DestinationArn: !GetAtt LogGroup.Arn + Format: ' $context.identity.sourceIp $context.identity.caller $context.identity.user $context.requestTime $context.httpMethod $context.resourcePath $context.status $context.protocol $context.responseLength$context.error.message' + EndpointConfiguration: + Type: REGIONAL + OpenApiVersion: '2.0' + StageName: live + DefinitionBody: + swagger: '2.0' + info: + title: 'geolocator' + schemes: + - 'https' + paths: + /: + get: + x-amazon-apigateway-integration: + type: aws_proxy + httpMethod: POST + uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${Geolocator.Arn}/invocations' + responses: {} + geolocatorS3Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub "webpresence-geolocator-${Environment}" + + LogGroup: + Type: Custom::LogGroup + Properties: + ServiceToken: !ImportValue LogGroupHelperLambdaArn + LogGroupName: !Sub '/${Environment}/webpresence/geolocator' + RetentionInDays: 3653 diff --git a/docs/cloudformation/search-demo.yml b/docs/cloudformation/search-demo.yml new file mode 100644 index 00000000..d9ffcbbb --- /dev/null +++ b/docs/cloudformation/search-demo.yml @@ -0,0 +1,256 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: patterns-s3-docrepo + +Parameters: + DocumentsBucketName: + Type: String + Default: 'patterns-s3docrepo-documents' + StagingBucketName: + Type: String + Default: 'patterns-s3docrepo-staging' + QueuedBucketName: + Type: String + Default: 'patterns-s3docrepo-queued' + ESdomain: + Type: String + Default: '<>' + +Globals: + Function: + Timeout: 15 + Environment: + Variables: + language: 'en' + +Resources: + ## S3 buckets + DocumentsBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref DocumentsBucketName + StagingBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref StagingBucketName + QueuedBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref QueuedBucketName + + ## SQS queue + MySqsQueue: + Type: AWS::SQS::Queue + + # Lambda functions + + queryESFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: queryESindex/ + Handler: app.handler + Runtime: nodejs12.x + MemorySize: 128 + Timeout: 10 + ReservedConcurrentExecutions: 1 + Environment: + Variables: + domain: !Ref ESdomain + Events: + HttpApiEvent: + Type: HttpApi + Properties: + Path: / + Method: GET + + AddToESFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: addToESindex/ + Handler: app.handler + Runtime: nodejs12.x + MemorySize: 128 + Timeout: 10 + ReservedConcurrentExecutions: 1 + Environment: + Variables: + domain: !Ref ESdomain + Policies: + - S3ReadPolicy: + BucketName: !Ref QueuedBucketName + - Statement: + - Effect: Allow + Resource: '*' + Action: + - comprehend:* + Events: + MySQSEvent: + Type: SQS + Properties: + Queue: !GetAtt MySqsQueue.Arn + BatchSize: 1 + + BatchingFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: batchingFunction/ + Handler: app.handler + Runtime: nodejs12.x + MemorySize: 512 + Environment: + Variables: + OutputBucket: !Ref QueuedBucketName + Policies: + - S3CrudPolicy: + BucketName: !Ref QueuedBucketName + - S3ReadPolicy: + BucketName: !Ref StagingBucketName + Events: + OriginalTextUpload: + Type: S3 + Properties: + Bucket: !Ref StagingBucket + Events: s3:ObjectCreated:* + Filter: + S3Key: + Rules: + - Name: suffix + Value: '.txt' + + AddToQueueFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: addToQueueFunction/ + Handler: app.handler + Runtime: nodejs12.x + Timeout: 15 + MemorySize: 128 + Environment: + Variables: + SQSqueueName: !Ref MySqsQueue + Policies: + - SQSSendMessagePolicy: + QueueName: !GetAtt MySqsQueue.QueueName + - S3ReadPolicy: + BucketName: !Ref QueuedBucketName + + Events: + FileUpload: + Type: S3 + Properties: + Bucket: !Ref QueuedBucket + Events: s3:ObjectCreated:* + + ProcessJPGFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: processJPG/ + Handler: app.handler + Runtime: nodejs12.x + MemorySize: 128 + Environment: + Variables: + OutputBucket: !Ref QueuedBucketName + MaxLabels: 5 + MinConfidence: 60 + Policies: + - S3CrudPolicy: + BucketName: !Ref QueuedBucketName + - S3ReadPolicy: + BucketName: !Ref DocumentsBucketName + - Statement: + - Effect: Allow + Resource: '*' + Action: + - rekognition:* + Events: + Upload: + Type: S3 + Properties: + Bucket: !Ref DocumentsBucket + Events: s3:ObjectCreated:* + Filter: + S3Key: + Rules: + - Name: suffix + Value: '.jpg' + + ProcessDOCXFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: processDOCX/ + Handler: app.handler + Runtime: nodejs12.x + MemorySize: 512 + Environment: + Variables: + OutputBucket: !Ref StagingBucketName + Policies: + - S3CrudPolicy: + BucketName: !Ref StagingBucketName + - S3ReadPolicy: + BucketName: !Ref DocumentsBucketName + Events: + Upload: + Type: S3 + Properties: + Bucket: !Ref DocumentsBucket + Events: s3:ObjectCreated:* + Filter: + S3Key: + Rules: + - Name: suffix + Value: '.docx' + + ProcessPDFFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: processPDF/ + Handler: app.handler + Runtime: nodejs12.x + MemorySize: 512 + Environment: + Variables: + OutputBucket: !Ref StagingBucketName + Policies: + - S3CrudPolicy: + BucketName: !Ref StagingBucketName + - S3ReadPolicy: + BucketName: !Ref DocumentsBucketName + Events: + Upload: + Type: S3 + Properties: + Bucket: !Ref DocumentsBucket + Events: s3:ObjectCreated:* + Filter: + S3Key: + Rules: + - Name: suffix + Value: '.pdf' + +## Outputs +Outputs: + DocumentsBucketName: + Description: Documents Bucket Name + Value: !Ref DocumentsBucket + + StagingBucketName: + Description: Staging Bucket Name + Value: !Ref StagingBucket + + QueuedBucketName: + Description: Queued Bucket Name + Value: !Ref QueuedBucket + + AddToESFunctionARN: + Description: This AddToESFunction function ARN. + Value: !GetAtt AddToESFunction.Arn + + AddToESFunctionRoleARN: + Description: This ARN needs permission in the Elasticsearch configuration. + Value: !GetAtt AddToESFunctionRole.Arn + + queryESFunctionARN: + Description: This ARN needs permission in the Elasticsearch configuration. + Value: !GetAtt queryESFunction.Arn \ No newline at end of file diff --git a/docs/cloudformation/tiler-web-presence.yml b/docs/cloudformation/tiler-web-presence.yml new file mode 100644 index 00000000..7cfd1385 --- /dev/null +++ b/docs/cloudformation/tiler-web-presence.yml @@ -0,0 +1,373 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: 'AWS::Serverless-2016-10-31' +Description: Deploys titiler for web presence + +Parameters: + Environment: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/environment + Description: SSM parameter name for environment + DeploymentBucket: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/deployment-bucket + Description: S3 bucket where all deployment files are stored + SslCertArn: + Type: AWS::SSM::Parameter::Value + Default: /webpresence/titiler/ssl-cert-arn + Description: SSM parameter name for titiler ACM SSL Cert ARN + WebAclArn: + Type: String + Description: ARN of the WAF web ACL to use for CloudFront + SecHeadersLambdaEdgeArn: + Type: String + Description: ARN of the Lambda@Edge function for injecting security headers to CloudFront + Bucket: + Type: CommaDelimitedList + Default: "datacube-prod-data-public" + Description: Specific S3 Buckets containing COGS that we want to tiled + DisableCOG: + Type: String + Default: "false" + AllowedValues: + - "true" + - "false" + DisableMosaic: + Type: String + Default: "true" + AllowedValues: + - "true" + - "false" + DisableSTAC: + Type: String + Default: "true" + AllowedValues: + - "true" + - "false" + +Conditions: + IsProd: !Equals [prod, !Ref Environment] + IsStage: !Equals [stage, !Ref Environment] + IsDev: !Equals [dev, !Ref Environment] + +Resources: + TiTiler: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.10 + Policies: + - AWSLambdaExecute # Managed Policy + - Version: '2012-10-17' # Policy Document + Statement: + - Effect: Allow + Action: + - s3:GetObject + - s3:HeadObject + Resource: + !Split + - ',' + - !Join + - '' + - - 'arn:aws:s3:::' + - !Join + - '/*,arn:aws:s3:::' + - !Ref Bucket + - '/*' + + InlineCode: | + import logging + from mangum import Mangum + from titiler.application.main import app + + logging.getLogger("mangum.lifespan").setLevel(logging.ERROR) + logging.getLogger("mangum.http").setLevel(logging.ERROR) + + handler = Mangum(app, lifespan="auto") + + MemorySize: 3009 + Handler: index.handler + Timeout: 90 + Environment: + Variables: + CPL_VSIL_CURL_ALLOWED_EXTENSIONS: '.tif,.TIF,.tiff' + GDAL_CACHEMAX: 200 + GDAL_DISABLE_READDIR_ON_OPEN: EMPTY_DIR + GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: YES + GDAL_HTTP_MULTIPLEX: YES + GDAL_HTTP_VERSION: 2 + VSI_CACHE: TRUE + VSI_CACHE_SIZE: 536870912 + CPL_VSIL_CURL_CACHE_SIZE: 200000000 + GDAL_INGESTED_BYTES_AT_OPEN: 32768 + TITILER_API_DISABLE_COG: !Ref DisableCOG + TITILER_API_DISABLE_STAC: !Ref DisableSTAC + TITILER_API_DISABLE_MOSAIC: !Ref DisableMosaic + Events: + ApiEvent: + Type: Api + Properties: + RestApiId: !Ref RestApi + Path: /* + Method: any + Layers: + - arn:aws:lambda:ca-central-1:552819999234:layer:titiler:8 + + LambdaImageProcessor: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.10 + Role: !GetAtt LambdaExecutionRole.Arn + CodeUri: + Bucket: !Ref DeploymentBucket + Key: + Fn::If: + - IsProd + - cloudformation-templates/lambda/image-processor/image-processor-20240619-1400.zip + - Fn::If: + - IsStage + - cloudformation-templates/lambda/image-processor/image-processor-20240619-1400.zip + - cloudformation-templates/lambda/image-processor/image-processor-20240619-1400.zip + MemorySize: 3009 + Handler: app.lambda_handler + Timeout: 60 + Events: + ApiEvent: + Type: Api + Properties: + RestApiId: !Ref RestApi + Path: /image-processor + Method: any + Layers: + - arn:aws:lambda:ca-central-1:770693421928:layer:Klayers-p310-Pillow:7 + - arn:aws:lambda:ca-central-1:770693421928:layer:Klayers-p310-requests:11 + + + LambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + + VerificationHeaderSecret: + Type: AWS::SecretsManager::Secret + Properties: + Description: Verification header secret value used to identify request coming from CloudFront to API Gateway + Name: /webpresence/titiler/verification-header-secret + GenerateSecretString: + ExcludePunctuation: true + PasswordLength: 32 + + OriginRequestPolicy: + Type: AWS::CloudFront::OriginRequestPolicy + Properties: + OriginRequestPolicyConfig: + CookiesConfig: + CookieBehavior: none + HeadersConfig: + HeaderBehavior: whitelist + Headers: ["Accept"] + Name: titiler-cloudfront-to-api-gateway + QueryStringsConfig: + QueryStringBehavior: all + + TiTilerDistribution: + Type: AWS::CloudFront::Distribution + DependsOn: VerificationHeaderSecret + Properties: + DistributionConfig: + Enabled: 'true' + Origins: + - Id: ApiOrigin + DomainName: !Sub ${RestApi}.execute-api.${AWS::Region}.amazonaws.com + CustomOriginConfig: + HTTPSPort: 443 + OriginProtocolPolicy: https-only + OriginSSLProtocols: + - TLSv1.2 + OriginPath: /live + OriginCustomHeaders: + - HeaderName: X-Origin-Verify + HeaderValue: '{{resolve:secretsmanager:/webpresence/titiler/verification-header-secret}}' + DefaultCacheBehavior: + AllowedMethods: [GET, HEAD, OPTIONS, PUT, PATCH, POST, DELETE] + ViewerProtocolPolicy: redirect-to-https + TargetOriginId: ApiOrigin + CachePolicyId: 4135ea2d-6df8-44a3-9df3-4b5a84be39ad # https://docs.amazonaws.cn/en_us/AmazonCloudFront/latest/DeveloperGuide/using-managed-cache-policies.html + OriginRequestPolicyId: !GetAtt OriginRequestPolicy.Id + LambdaFunctionAssociations: + - EventType: origin-response + LambdaFunctionARN: !Ref SecHeadersLambdaEdgeArn + HttpVersion: http2 + WebACLId: !Ref WebAclArn + Logging: + Bucket: !Sub 'webpresence-cloudfront-access-logs-${Environment}.s3.amazonaws.com' + IncludeCookies: true + Prefix: titiler + Aliases: + - !If + - IsProd + - tiler.api.geo.ca + - !Sub 'tiler-${Environment}.api.geo.ca' + - !If + - IsProd + - www.tiler.api.geo.ca + - !Sub 'www.tiler-${Environment}.api.geo.ca' + PriceClass: PriceClass_All + ViewerCertificate: + AcmCertificateArn: !Ref SslCertArn + SslSupportMethod: sni-only + MinimumProtocolVersion: TLSv1.2_2021 + + RestApiWebAcl: + Type: AWS::WAFv2::WebACL + DependsOn: VerificationHeaderSecret + Properties: + Name: webpresence-titiler-cloudfront + Scope: REGIONAL + Description: "Restrict access to CloudFront" + DefaultAction: + Block: {} + VisibilityConfig: + SampledRequestsEnabled: true + CloudWatchMetricsEnabled: true + MetricName: webpresence-titiler-cloudfront + Rules: + - Name: webpresence-titiler-cloudfront-header-verification + Priority: 0 + Action: + Allow: {} + VisibilityConfig: + SampledRequestsEnabled: true + CloudWatchMetricsEnabled: true + MetricName: webpresence-titiler-cloudfront-header-verification + Statement: + ByteMatchStatement: + FieldToMatch: + SingleHeader: + Name: X-Origin-Verify + PositionalConstraint: EXACTLY + SearchString: '{{resolve:secretsmanager:/webpresence/titiler/verification-header-secret}}' + TextTransformations: + - Priority: 0 + Type: NONE + + RestApiWebAclAssociations: + Type: AWS::WAFv2::WebACLAssociation + DependsOn: + - RestApiliveStage + Properties: + ResourceArn: !Sub 'arn:aws:apigateway:${AWS::Region}::/restapis/${RestApi}/stages/live' + WebACLArn: !GetAtt RestApiWebAcl.Arn + + RestApi: + Type: AWS::Serverless::Api + Properties: + Name: titiler + AccessLogSetting: + DestinationArn: !GetAtt LogGroup.Arn + Format: ' $context.identity.sourceIp $context.identity.caller $context.identity.user $context.requestTime $context.httpMethod $context.resourcePath $context.status $context.protocol $context.responseLength$context.error.message' + EndpointConfiguration: + Type: REGIONAL + OpenApiVersion: '2.0' + StageName: live + BinaryMediaTypes: ["*/*","application/vnd.oai.openapi+json"] + DefinitionBody: + swagger: '2.0' + info: + title: 'titiler' + schemes: + - 'https' + paths: + /: + get: + x-amazon-apigateway-integration: + type: aws_proxy + httpMethod: POST + uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${TiTiler.Arn}/invocations' + responses: {} + post: + x-amazon-apigateway-integration: + type: aws_proxy + httpMethod: POST + uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${TiTiler.Arn}/invocations' + responses: {} + /{proxy+}: + get: + x-amazon-apigateway-integration: + type: aws_proxy + httpMethod: POST + uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${TiTiler.Arn}/invocations' + responses: {} + post: + x-amazon-apigateway-integration: + type: aws_proxy + httpMethod: POST + uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${TiTiler.Arn}/invocations' + responses: {} + /image-processor: + get: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + x-amazon-apigateway-integration: + uri: !Sub "arn:aws:apigateway:ca-central-1:lambda:path/2015-03-31/functions/arn:aws:lambda:ca-central-1:${AWS::AccountId}:function:${LambdaImageProcessor}/invocations" + httpMethod: "POST" + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Origin: "'*'" + type: "aws_proxy" + options: + consumes: + - "application/json" + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + headers: + Access-Control-Allow-Origin: + type: "string" + Access-Control-Allow-Methods: + type: "string" + Access-Control-Allow-Headers: + type: "string" + x-amazon-apigateway-integration: + responses: + default: + statusCode: "200" + responseParameters: + method.response.header.Access-Control-Allow-Methods: "'GET,OPTIONS'" + method.response.header.Access-Control-Allow-Headers: "'Content-Type,Authorization,X-Amz-Date,X-Api-Key,X-Amz-Security-Token'" + method.response.header.Access-Control-Allow-Origin: "'*'" + requestTemplates: + application/json: "{\"statusCode\": 200}" + passthroughBehavior: "when_no_match" + type: "mock" + + LogGroup: + Type: Custom::LogGroup + Properties: + ServiceToken: !ImportValue LogGroupHelperLambdaArn + LogGroupName: !Sub '/${Environment}/webpresence/titiler' + RetentionInDays: 3653 diff --git a/docs/geocore-format-template.json b/docs/geocore-format-template.json new file mode 100644 index 00000000..84648af7 --- /dev/null +++ b/docs/geocore-format-template.json @@ -0,0 +1,130 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type":"Feature", + "geometry": { + "type": null, + "coordinates": [ + [ + [null , null], [null , null], [null , null], [null , null], [null , null] + ]] + }, + "properties": { + "id": null, + "title": { + "en": null, + "fr": null + }, + "description": { + "en": null, + "fr": null + }, + "keywords": { + "en": null, + "fr": null + }, + "topicCategory": null, + "parentIdentifier": null, + "date": { + "published": { + "text": null, + "date": null + }, + "created": { + "text": null, + "date": null + }, + "revision": { + "text": null, + "date": null + }, + "notavailable": { + "text": null, + "date": null + }, + "inforce": { + "text": null, + "date": null + }, + "adopted": { + "text": null, + "date": null + }, + "deprecated": { + "text": null, + "date": null + }, + "superceded": { + "text": null, + "date": null + } + }, + "spatialRepresentation": null, + "type": null, + "geometry": null, + "temporalExtent": { + "begin": null , + "end": null + }, + "refSys": null, + "refSys_version": null, + "status": null, + "maintenance": null, + "metadataStandard": { + "en": null, + "fr": null + }, + "metadataStandardVersion": null, + "graphicOverview": [ + { + "overviewFileName": null, + "overviewFileDescription": null, + "overviewFileTupe": null + }], + "distributionFormat_name": null, + "distributionFormat_format": null , + "useLimits": { + "en": null, + "fr": null + }, + "accessConstraints": null, + "otherConstraints": { + "en": null, + "fr": null + }, + "dateStamp": null, + "dataSetURI": null, + "locale": { + "language": null, + "country": null, + "encoding": null + }, + "language": null, + "characterSet": null, + "environmentDescription": null, + "supplementalInformation": { + "en": null, + "fr": null + }, + "contact": [null], + "credits": [null], + "cited": [null], + "distributor": [null], + "options": [ + { + "url":null, + "protocol":null, + "name": { + "en":null, + "fr":null + } + }], + "sourceSystemName":null, + "eoCollection": null, + "eoFilters":[null] + } + }] + } + + \ No newline at end of file diff --git a/docs/lambda/collections/collections-20230529-1500.zip b/docs/lambda/collections/collections-20230529-1500.zip new file mode 100644 index 00000000..34fcdbd9 Binary files /dev/null and b/docs/lambda/collections/collections-20230529-1500.zip differ diff --git a/docs/lambda/collections/collections-20230926-1100.zip b/docs/lambda/collections/collections-20230926-1100.zip new file mode 100644 index 00000000..6b7e4651 Binary files /dev/null and b/docs/lambda/collections/collections-20230926-1100.zip differ diff --git a/docs/lambda/collections/collections-20240612-1600.zip b/docs/lambda/collections/collections-20240612-1600.zip new file mode 100644 index 00000000..98cbcd22 Binary files /dev/null and b/docs/lambda/collections/collections-20240612-1600.zip differ diff --git a/docs/lambda/featured/featured.zip b/docs/lambda/featured/featured-220323.zip similarity index 100% rename from docs/lambda/featured/featured.zip rename to docs/lambda/featured/featured-220323.zip diff --git a/docs/lambda/geo/geo-20230330-0000.zip b/docs/lambda/geo/geo-20230330-0000.zip new file mode 100644 index 00000000..8c5cf415 Binary files /dev/null and b/docs/lambda/geo/geo-20230330-0000.zip differ diff --git a/docs/lambda/geo/geo-20230612-1700.zip b/docs/lambda/geo/geo-20230612-1700.zip new file mode 100644 index 00000000..39b0ff90 Binary files /dev/null and b/docs/lambda/geo/geo-20230612-1700.zip differ diff --git a/docs/lambda/geo/geo-20231218-1700.zip b/docs/lambda/geo/geo-20231218-1700.zip new file mode 100644 index 00000000..c3c62d9c Binary files /dev/null and b/docs/lambda/geo/geo-20231218-1700.zip differ diff --git a/docs/lambda/geo/geo-20240627-2300.zip b/docs/lambda/geo/geo-20240627-2300.zip new file mode 100644 index 00000000..bf59179c Binary files /dev/null and b/docs/lambda/geo/geo-20240627-2300.zip differ diff --git a/docs/lambda/geo/geo.zip b/docs/lambda/geo/geo.zip deleted file mode 100644 index 55cd1951..00000000 Binary files a/docs/lambda/geo/geo.zip and /dev/null differ diff --git a/docs/lambda/geocore-eo-harvester/rcm-ard/geocore-to-parquet-20240610-1500.zip b/docs/lambda/geocore-eo-harvester/rcm-ard/geocore-to-parquet-20240610-1500.zip new file mode 100644 index 00000000..829ae04e Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm-ard/geocore-to-parquet-20240610-1500.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-aggregator-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-aggregator-20240610-2100.zip new file mode 100644 index 00000000..cc80483f Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-aggregator-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-collector-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-collector-20240610-2100.zip new file mode 100644 index 00000000..80d8089d Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-collector-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-gather-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-gather-20240610-2100.zip new file mode 100644 index 00000000..d07bc3ef Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-gather-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-processor-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-processor-20240610-2100.zip new file mode 100644 index 00000000..224cdfd9 Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-processor-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-scatter-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-scatter-20240610-2100.zip new file mode 100644 index 00000000..de4a3c69 Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm-ard/rcm-ard-sg-scatter-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm/geocore-to-parquet-20240610-1500.zip b/docs/lambda/geocore-eo-harvester/rcm/geocore-to-parquet-20240610-1500.zip new file mode 100644 index 00000000..829ae04e Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm/geocore-to-parquet-20240610-1500.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-aggregator-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-aggregator-20240610-2100.zip new file mode 100644 index 00000000..cc80483f Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-aggregator-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-collector-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-collector-20240610-2100.zip new file mode 100644 index 00000000..80d8089d Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-collector-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-gather-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-gather-20240610-2100.zip new file mode 100644 index 00000000..d07bc3ef Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-gather-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-processor-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-processor-20240610-2100.zip new file mode 100644 index 00000000..224cdfd9 Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-processor-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-scatter-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-scatter-20240610-2100.zip new file mode 100644 index 00000000..de4a3c69 Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/rcm/rcm-sg-scatter-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/sentinel-1/geocore-to-parquet-20240610-1500.zip b/docs/lambda/geocore-eo-harvester/sentinel-1/geocore-to-parquet-20240610-1500.zip new file mode 100644 index 00000000..829ae04e Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/sentinel-1/geocore-to-parquet-20240610-1500.zip differ diff --git a/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-aggregator-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-aggregator-20240610-2100.zip new file mode 100644 index 00000000..cc80483f Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-aggregator-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-collector-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-collector-20240610-2100.zip new file mode 100644 index 00000000..80d8089d Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-collector-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-gather-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-gather-20240610-2100.zip new file mode 100644 index 00000000..d07bc3ef Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-gather-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-processor-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-processor-20240610-2100.zip new file mode 100644 index 00000000..224cdfd9 Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-processor-20240610-2100.zip differ diff --git a/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-scatter-20240610-2100.zip b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-scatter-20240610-2100.zip new file mode 100644 index 00000000..de4a3c69 Binary files /dev/null and b/docs/lambda/geocore-eo-harvester/sentinel-1/sentinel1-sg-scatter-20240610-2100.zip differ diff --git a/docs/lambda/geocore-to-parquet/geocore-to-parquet.zip b/docs/lambda/geocore-filters/popularity-20230330-0000.zip similarity index 98% rename from docs/lambda/geocore-to-parquet/geocore-to-parquet.zip rename to docs/lambda/geocore-filters/popularity-20230330-0000.zip index 27127724..46ad563e 100644 Binary files a/docs/lambda/geocore-to-parquet/geocore-to-parquet.zip and b/docs/lambda/geocore-filters/popularity-20230330-0000.zip differ diff --git a/docs/lambda/geocore-filters/popularity-20230330-0800.zip b/docs/lambda/geocore-filters/popularity-20230330-0800.zip new file mode 100644 index 00000000..00126ecb Binary files /dev/null and b/docs/lambda/geocore-filters/popularity-20230330-0800.zip differ diff --git a/docs/lambda/geocore-hnap-harvest/geocore-hnap-harvest.zip b/docs/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230330-0000.zip similarity index 96% rename from docs/lambda/geocore-hnap-harvest/geocore-hnap-harvest.zip rename to docs/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230330-0000.zip index 1b23bcef..f4a67b1f 100644 Binary files a/docs/lambda/geocore-hnap-harvest/geocore-hnap-harvest.zip and b/docs/lambda/geocore-hnap-harvest/geocore-hnap-harvest-20230330-0000.zip differ diff --git a/docs/lambda/geocore-to-parquet/geocore-to-parquet-20230330-1500.zip b/docs/lambda/geocore-to-parquet/geocore-to-parquet-20230330-1500.zip new file mode 100644 index 00000000..203f576a Binary files /dev/null and b/docs/lambda/geocore-to-parquet/geocore-to-parquet-20230330-1500.zip differ diff --git a/docs/lambda/geocore-to-parquet/geocore-to-parquet-20230823-1500.zip b/docs/lambda/geocore-to-parquet/geocore-to-parquet-20230823-1500.zip new file mode 100644 index 00000000..d423c500 Binary files /dev/null and b/docs/lambda/geocore-to-parquet/geocore-to-parquet-20230823-1500.zip differ diff --git a/docs/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip b/docs/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip new file mode 100644 index 00000000..829ae04e Binary files /dev/null and b/docs/lambda/geocore-to-parquet/geocore-to-parquet-20240610-1500.zip differ diff --git a/docs/lambda/geocore-transform/geocore-transform.zip b/docs/lambda/geocore-transform/geocore-transform-20221114-1700.zip similarity index 93% rename from docs/lambda/geocore-transform/geocore-transform.zip rename to docs/lambda/geocore-transform/geocore-transform-20221114-1700.zip index 1ec178ed..291fe159 100644 Binary files a/docs/lambda/geocore-transform/geocore-transform.zip and b/docs/lambda/geocore-transform/geocore-transform-20221114-1700.zip differ diff --git a/docs/lambda/geolocator/README.md b/docs/lambda/geolocator/README.md new file mode 100644 index 00000000..d15d82ed --- /dev/null +++ b/docs/lambda/geolocator/README.md @@ -0,0 +1,10 @@ +# README + +## Deployment steps + +- the content of the `geolocator-bucket-content` folder must be deployed manually the the `webpresence-geolocator-${Environment}` bucket. the lambda will read it's config from there at runtime. + +## Deployment quirks + +- the lambda name needs to change on every release to be correctly picked up by the cloudformation. this name change should be also reflected in the `geolocator-web-presence` cloudformation. +- this lambda function references files from an s3 bucket at runtime. Theses files need to be deployed manually when geolocator is updated. The name of the bucket is specified in the `geolocator-web-presence` cloudformation. diff --git a/docs/lambda/geolocator/geolocator-bucket-content/README.md b/docs/lambda/geolocator/geolocator-bucket-content/README.md new file mode 100644 index 00000000..8a2af130 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/README.md @@ -0,0 +1,240 @@ +# Schemas and Metadata Structure + +Geolocator API is based on 'schemas' to define the way the application interacts with data. Different schemas are required to define the following tasks: + • Read and validate the parameters passed to the API-geolocator application. This include the keys (the available services), the query and the language (when it's needed) + + • Build valid url and parameters for consuming data on each specific geo-service + + • Extract the required information from the query and build standard response from it + + • Validate that the extracted data falls into valid parameters (mainly numeric values) + + +### Geolocator API Schemas +The Geolocator API has schemas for [input](./api/in-api-schema.json), [output](./api-out-schema.json) and several specific services schemas [./services/__service_id__-schema.json] + +The __input__ schema identifies the expected parameters to query the API. + - "q": The query to parse and send to supported API's. + - "lang": The language on wich to filter the query (fr or en). + - "keys": The list of supported API key to query. Optional parameter, if missing, all + supported key will be queryied. Every time we support a new API or services, a new key will be added to this array of accepted values. + - "dev": Show null or undefined fields with key and table name if applicable (true or false) + - "table": optional parameter to return json dump of table created from csv files. + Default: none + Values: category, component, province, generic and tableurl + +The __output__ schemas validates the obtained values match type and limits for each field. + - "key": The service Id for the source of the data + - "name": The generic name of the feature. + - "lat": The latitude coordinate. + - "lng": The longitude coordinate. + - "bbox": The bbox [minX, minY, maxX, maxY]. + - "province": The province the item belongs to. Optional return value. + - "tag": The tag value of the item. Optional return value. tags may be different from one API to + the other, it is a value to help understand what type of item it is. + It also includes a key ["properties"] with the list of valid 'services'. + +each __service_id__ schema contains the rules to extract, adapt and validate the load data obtained after querying from that specific service. + +### Supported API and services Metadata +Each supported APIs and services may have differents input and output signatures. To help the parsing of these signatures, the Geolocator API will rely on JSON metadata file. This metadata file will also holds connection information like urls. The name of this file is the value of the key item (<__service_id__>-schema.json). + +The structure of this files is: +``` +{ + "url": "https://.../_PARAM1_/...?", //API url with optional parameters + "urlParams": { + "param1": "lang" // Optional parameter to substitue in urls + }, + "staticParams": + "countrycodes=CA", // Fixed parameters required by the service, + "format=jsonv2" // they will be added to the url before execution + }, + "urlCodeTables": { // Additional urls required to extract data tables for specific fields + "province" : { + "url": "https://geogratis.gc.ca/services/geoname/en/codes/province.json", + // other keys/values here + } + } + "lookup": { + "in": { // Paramaters replacements specific for the current service + "q": "q", + "lang": "accept-language" + }, + "out": { // Output lookup information + "type": "array", // the type of data structure retrieved from the service + "items": { // Set of attributes to be fullfiled by the input data with + // specific rules each + "name": { + "field": "name", // Return JSON item to look for + "lookup": "" // Lookup to apply if needed + }, + "lat": { // field path to get access to the data value for this field + "field":"geometry.location.lat", + "lookup": "" + } + "bbox": { + "field": "bbox", + "lookup": "" + }, + "province": { + "field": "province.code", + "lookup": { // describes the structure and attributes to obtain the data value + "type": "table", + "field": "description" + } + }, + "tag": [ // Can contains many field values separated by comma ',' + // For optional parameter like "tag", the field value + // can be left empty if no value is found + { + "field": "location", + "lookup": "" + }, + { + "field": "generic.code", + "lookup": { + "type": "table", + "field": "term" + } + } + ] + } + } + } +} +``` + +### Lookup +Lookup can have several signatures: + * None. The field is directly accesible + * table. The data value must be extracted from a key-value table + * array. The data value comes from an specific position in an array of values + * search. The data should be found by searching in a data field containing + a list of dictionaries where one must match an specific key/value tuple + * average. Calculate the average of a subset of numeric values from a field + containing a list of numerics (such as bbox) + * url. The data value comes from a table obtained from a url (deprecated) + * csv. The data value is extracted from a csv (comma separated values) string + * type. The data value is extracted from different fields or tables depending on + the value in the type field +### Response time +The response time for the query depends on several factors: + - The service required. Different services perform different based on the + complexity to adapt the load to the standard output format. + Several services at once will ad up to the final delay. + - The size of the load. More items required more time to process. + The more specific the query, the fewer features will be processed. (it was + already improved by keeping a list of micro-schemas asociated to each field. + Even though the list must be cleared and rebuild for each service, the + computing time is close to O(N) instead of O(N2)). + - A delay of 5 minutes without performing any requests, will increase the + response time in about 3 seconds for the next one, due to + the life time of objects in memory. + + * An url to tests concurrencial calls can be used for performance validation. + https://3wdd7ausil.execute-api.ca-central-1.amazonaws.com/dev?iterations=1000 + + +### Google API's + To test Google services such as 'findplace' and 'geocode' without the key. + Simulate the return from the service by bypassing the url_request with a condition + and the expected returns. + ``` + https://maps.googleapis.com/maps/api/geocode/json?address=confederation%20park&key=AIzaSyASQcYTDCw4fRr_GY5WHxIAqeTsDmvAh_8 + ``` + * Add the service in in-api-schema.json: + ``` + "enum": ["geonames", "nominatim", "locate", "nts", "geocode"] + ``` + * Add the service in out-api-schema.json: + ``` + "properties": { + "geocode": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/definitions/output" }, + "description": "The Geocode api result set." + }, + } + ``` + * geolocator_lambda.py should be 'temporarily' modified in this way: + ``` + if service_id != "geocode": + service_load = url_request(url, params) + else: + service_load = { + "results" : [ + { + "address_components" : [ + { + "long_name" : "Confederation Park", + "short_name" : "Confederation Park", + "types" : [ "establishment", "park", "point_of_interest", "tourist_attraction" ] + }, + { + "long_name" : "Elgin Street", + "short_name" : "Elgin St", + "types" : [ "route" ] + }, + { + "long_name" : "Downtown", + "short_name" : "Downtown", + "types" : [ "neighborhood", "political" ] + }, + { + "long_name" : "Ottawa", + "short_name" : "Ottawa", + "types" : [ "locality", "political" ] + }, + { + "long_name" : "Ottawa", + "short_name" : "Ottawa", + "types" : [ "administrative_area_level_2", "political" ] + }, + { + "long_name" : "Ontario", + "short_name" : "ON", + "types" : [ "administrative_area_level_1", "political" ] + }, + { + "long_name" : "Canada", + "short_name" : "CA", + "types" : [ "country", "political" ] + }, + { + "long_name" : "K1P 5J2", + "short_name" : "K1P 5J2", + "types" : [ "postal_code" ] + } + ], + "formatted_address" : "Confederation Park, Elgin St, Ottawa, ON K1P 5J2, Canada", + "geometry" : { + "location" : { + "lat" : 45.422394, + "lng" : -75.692452 + }, + "location_type" : "GEOMETRIC_CENTER", + "viewport" : { + "northeast" : { + "lat" : 45.4235181802915, + "lng" : -75.69165646970849 + }, + "southwest" : { + "lat" : 45.4208202197085, + "lng" : -75.69435443029151 + } + } + }, + "partial_match" : true, + "place_id" : "ChIJ55USI_mq0kwR0mbQ4LTQyU4", + "plus_code" : { + "compound_code" : "C8C5+X2 Somerset Ward, Ottawa, ON, Canada", + "global_code" : "87Q6C8C5+X2" + }, + "types" : [ "establishment", "park", "point_of_interest", "tourist_attraction" ] + } + ], + "status" : "OK" + } + ``` diff --git a/docs/lambda/geolocator/geolocator-bucket-content/api/in-api-schema.json b/docs/lambda/geolocator/geolocator-bucket-content/api/in-api-schema.json new file mode 100644 index 00000000..017e3716 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/api/in-api-schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "API Geolocator Input Config Schema", + "type": "object", + "version": 1.0, + "comments": "Schema for GeoView-API-geolocator input-query. The items inside parameters properties are the parameters provided by the URL.", + + "properties": { + "params": { + "type": "object", + "description": "Query parameter coming from the API call url", + "properties": { + "querystring": { + "type": "array", + "description": "Dictionary containing the parameters and values passed with the url", + "properties": { + "q": { + "type": "string", + "default": "none", + "description": "The query to parse and send to supported API's." + }, + "lang": { + "type": "string", + "enum": ["en", "fr"], + "default" : "en", + "description": "The language on wich to filter the query." + }, + "dev": { + "type": "string", + "enum": ["true", "false"], + "default" : "false", + "description": "show fields that are null or undefined" + }, + "table": { + "type": "string", + "enum": ["category", "component", "generic", "province", "none"], + "default" : "none", + "description": "return json dump of table" + }, + "keys": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": ["geonames", "nominatim", "locate"] + }, + "description": "The list of supported API key to query. Optional parameter, if missing, all supported key will be query." + } + }, + "required": ["q","table"], + "requiredAll": "false", + "additionalProperties": false + } + }, + "required": ["querystring"], + "additionalProperties": false + } + }, + "required": ["params"], + "additionalProperties": false +} diff --git a/docs/lambda/geolocator/geolocator-bucket-content/api/out-api-schema.json b/docs/lambda/geolocator/geolocator-bucket-content/api/out-api-schema.json new file mode 100644 index 00000000..05e7bfa2 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/api/out-api-schema.json @@ -0,0 +1,113 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "API Geolocator Output Config Schema", + "type": "object", + "version": 1.0, + "comments": "Schema for GeoView-API-geolocator output-query results.", + "additionalProperties": false, + + "definitions": { + "output": { + "type": "array", + "description": "Array of results were each item is the output from the requested API key.", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "Origin API key value." + }, + "name": { + "type": "string", + "description": "Name information of the item." + }, + "lat": { + "type": "number", + "minimum": -90, + "maximum": 90, + "description": "Latitude of the item." + }, + "lng": { + "type": "number", + "minimum": -180, + "maximum": 180, + "description": "Longitude of the item." + }, + "bbox": { + "type": "array", + "minItems": 4, + "items": [ + { + "type": "number", + "minimum": -180, + "maximum": 180, + "description": "West border of the bounding box." + }, + { + "type": "number", + "minimum": -90, + "maximum": 90, + "description": "South border of the bounding box." + }, + { + "type": "number", + "minimum": -180, + "maximum": 180, + "description": "East border of the bounding box." + }, + { + "type": "number", + "minimum": -90, + "maximum": 90, + "description": "North border of the bounding box." + } + ], + "description": "The bbox of the item [minX, minY, maxX, maxY]. Optional return value." + }, + "province": { + "type": "string", + "description": "The province the item belongs to. Optional return value, may be derived from the name parameter." + }, + "tag": { + "type": "array", + "minItems": 0, + "items": { + "type": "string", + "description": "The content of this field might vary depending of the sources." + }, + "description": "The tag value of the item. Optional return value. tags may be different from one API to the other, it is a value to help understand what type of item it is." + } + }, + "required": ["name", "lat", "lng"], + "additionalProperties": false + } + } + }, + "properties": { + "geonames": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/definitions/output" }, + "description": "The Geoname api result set." + }, + "nominatim": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/definitions/output" }, + "description": "The Nominatim api result set." + }, + "locate": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/definitions/output" }, + "description": "The locate api result set." + }, + "nts": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/definitions/output" }, + "description": "The ens api result set." + } + }, + "description": "List of supported key for the API. As we add key, we need to had an entry in the schema. The entry MUST be the same key as the in in-api-schema.json (keys enum parameter)." +} \ No newline at end of file diff --git a/docs/lambda/geolocator/geolocator-bucket-content/services/README.md b/docs/lambda/geolocator/geolocator-bucket-content/services/README.md new file mode 100644 index 00000000..c2604ab7 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/services/README.md @@ -0,0 +1,9 @@ +# in-api-schema keys parameter + +valid keys: geonames, nominatim and locate. + +nts key (https://geogratis.gc.ca/services/delimitation/en/nts) not used as there is no query parameter (only search by spatial bbox). +Static table and filtering capacity would need to be implemented. + + + diff --git a/docs/lambda/geolocator/geolocator-bucket-content/services/findplace-schema.json b/docs/lambda/geolocator/geolocator-bucket-content/services/findplace-schema.json new file mode 100644 index 00000000..ce5d5b78 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/services/findplace-schema.json @@ -0,0 +1,55 @@ +{ + "url" : "https://maps.googleapis.com/maps/api/place/findplacefromtext/json", + + "urlParams" : { + }, + "staticParams" : { + "inputtype" : "textquery", + "fields" : "name,geometry", + "key" : "AIzaSyASQcYTDCw4fRr_GY5WHxIAqeTsDmvAh_8" + }, + "urlCodeTables" : {}, + "lookup" : { + "in" : { + "q" : "input" + }, + "out" : { + "structure" : { + "type": "dict", + "key": "candidates" + }, + "data" : { + "name": { + "field": "name", + "lookup": "" + }, + "lat": { + "field":"geometry.location.lat", + "lookup": "" + }, + "lng": { + "field": "geometry.location.lng", + "lookup": "" + }, + "bbox" : [ + { + "field": "geometry.viewport.southwest.lng", + "lookup": "" + }, + { + "field": "geometry.viewport.southwest.lat", + "lookup": "" + }, + { + "field": "geometry.viewport.northeast.lng", + "lookup": "" + }, + { + "field": "geometry.viewport.northeast.lat", + "lookup": "" + } + ] + } + } + } +} diff --git a/docs/lambda/geolocator/geolocator-bucket-content/services/geocode-schema.json b/docs/lambda/geolocator/geolocator-bucket-content/services/geocode-schema.json new file mode 100644 index 00000000..80afa71b --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/services/geocode-schema.json @@ -0,0 +1,67 @@ +{ + "url" : "https://maps.googleapis.com/maps/api/geocode/json", + + "urlParams" : { + }, + "staticParams" : { + "key" : "AIzaSyASQcYTDCw4fRr_GY5WHxIAqeTsDmvAh_8" + }, + "urlCodeTables" : {}, + "lookup" : { + "in" : { + "q" : "address" + }, + "out" : { + "structure" : { + "type": "dict", + "key": "results" + }, + "data" : { + "name": { + "field": "formatted_address", + "lookup": "" + }, + "lat": { + "field":"geometry.location.lat", + "lookup": "" + }, + "lng": { + "field": "geometry.location.lng", + "lookup": "" + }, + "bbox" : [ + { + "field": "geometry.viewport.southwest.lng", + "lookup": "" + }, + { + "field": "geometry.viewport.southwest.lat", + "lookup": "" + }, + { + "field": "geometry.viewport.northeast.lng", + "lookup": "" + }, + { + "field": "geometry.viewport.northeast.lat", + "lookup": "" + } + ], + "province": { + "field": "address_components", + "lookup": { + "type": "search", + "search_field": "types", + "contains": "administrative_area_level_1", + "return_field": "long_name" + } + }, + "tag": { + "field": "types", + "lookup": "" + } + + } + } + } +} diff --git a/docs/lambda/geolocator/geolocator-bucket-content/services/geonames-schema.json b/docs/lambda/geolocator/geolocator-bucket-content/services/geonames-schema.json new file mode 100644 index 00000000..084053f9 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/services/geonames-schema.json @@ -0,0 +1,76 @@ +{ + "url": "https://geogratis.gc.ca/services/geoname/_PARAM1_/geonames.json", + "urlParams": { + "param1": "lang" + }, + "staticParams": {}, + "urlCodeTables": { + "province" : { + "url": "https://geogratis.gc.ca/services/geoname/_PARAM1_/codes/province.json", + "type": "array", + "name": "definitions", + "fields": { + "code": "code", + "description": "description" + } + }, + "generic": { + "url": "https://geogratis.gc.ca/services/geoname/_PARAM1_/codes/generic.json", + "type": "array", + "name": "definitions", + "fields": { + "code": "code", + "description": "term" + } + } + }, + "lookup": { + "in": { + "q": "q" + }, + "out": { + "structure" : { + "type": "dict", + "key": "items" + }, + "data": { + "name": { + "field": "name", + "lookup": "" + }, + "province": { + "field": "province.code", + "lookup": { + "type": "table", + "field": "description" + } + }, + "category": { + "field": "generic.code", + "lookup": { + "type": "table", + "field": "term" + } + }, + "lat": { + "field":"latitude", + "lookup": "" + }, + "lng": { + "field": "longitude", + "lookup": "" + }, + "bbox": { + "field": "bbox", + "lookup": "" + }, + "tag": [ + { + "field": "location", + "lookup": "" + } + ] + } + } + } +} diff --git a/docs/lambda/geolocator/geolocator-bucket-content/services/locate-schema.json b/docs/lambda/geolocator/geolocator-bucket-content/services/locate-schema.json new file mode 100644 index 00000000..87365f91 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/services/locate-schema.json @@ -0,0 +1,67 @@ +{ + "url": "https://geogratis.gc.ca/services/geolocation/_PARAM1_/locate", + "urlParams": { + "param1": "lang" + }, + "staticParams": { + "expand": "component" + }, + "urlCodeTables": {}, + "lookup": { + "in": { + "q": "q" + }, + "out": { + "structure" : { + "type": "list" + }, + "data": { + "name": { + "field": "component.name", + "lookup": { + "type": "type", + "field": "name" + } + }, + "province": { + "field": "component.province", + "lookup": { + "type": "table", + "field": "description" + } + }, + "category": { + "field": "component.generic", + "lookup": { + "type": "type", + "field": "category" + } + }, + "lat": { + "field":"geometry.coordinates", + "lookup": { + "type": "array", + "field":"1" + } + }, + "lng": { + "field": "geometry.coordinates", + "lookup": { + "type": "array", + "field":"0" + } + }, + "bbox": { + "field": "bbox", + "lookup": "" + }, + "tag": [ + { + "field": "component.location", + "lookup": "" + } + ] + } + } + } +} diff --git a/docs/lambda/geolocator/geolocator-bucket-content/services/nominatim-schema.json b/docs/lambda/geolocator/geolocator-bucket-content/services/nominatim-schema.json new file mode 100644 index 00000000..c2d51a63 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/services/nominatim-schema.json @@ -0,0 +1,89 @@ +{ + "url": "https://nominatim.openstreetmap.org/search", + "urlParams": {}, + "staticParams": { + "countrycodes": "CA", + "format": "jsonv2" + }, + "urlCodeTables": {}, + "lookup": { + "in": { + "q": "q", + "lang": "accept-language" + }, + "out": { + "structure" : { + "Type": "list" + }, + "data": { + "name": { + "field": "display_name", + "lookup": { + "type": "csv", + "field": "name", + "range": "3" + } + }, + "province": { + "field": "display_name", + "lookup": { + "type": "csv", + "field": "province", + "range": "3" + } + }, + "category": { + "field": "category", + "lookup": { + "type": "table", + "field": "description" + } + }, + "lat": { + "field":"lat", + "lookup": "" + }, + "lng": { + "field": "lon", + "lookup": "" + }, + "bbox": [ + { + "field": "boundingbox", + "lookup": { + "type": "array", + "field":"2" + } + }, + { + "field": "boundingbox", + "lookup": { + "type": "array", + "field":"0" + } + }, + { + "field": "boundingbox", + "lookup": { + "type": "array", + "field":"3" + } + }, + { + "field": "boundingbox", + "lookup": { + "type": "array", + "field":"1" + } + } + ], + "tag": [ + { + "field": "category", + "lookup": "" + } + ] + } + } + } +} diff --git a/docs/lambda/geolocator/geolocator-bucket-content/services/nts-schema.json b/docs/lambda/geolocator/geolocator-bucket-content/services/nts-schema.json new file mode 100644 index 00000000..b2e24675 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/services/nts-schema.json @@ -0,0 +1,38 @@ +{ + "url": "https://geogratis.gc.ca/services/delimitation/_PARAM1_/nts", + "urlParams": { + "param1": "lang" + }, + "staticParams": {}, + "urlCodeTables": {}, + "lookup": { + "in": { + "q": "q" + }, + "out": { + "structure" : { + "type": "dict", + "key": "features" + }, + "data": { + "name": { + "field": "properties.identifier", + "lookup": "" + }, + "bbox": { + "field": "bbox", + "lookup": "" + }, + "province": { + "field": "" + }, + "tag": [ + { + "field": "properties.scale", + "lookup": "" + } + ] + } + } + } +} diff --git a/docs/lambda/geolocator/geolocator-bucket-content/tables/README.md b/docs/lambda/geolocator/geolocator-bucket-content/tables/README.md new file mode 100644 index 00000000..6b5a9e4a --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/tables/README.md @@ -0,0 +1,10 @@ +# Lookup tables configuration + +For the geonames key, 2 csv files (generic.csv, province.csv) contain generic location and province names. + +The generic.csv and province.csv files are loaded into tables, to reduce API calls to the geonames service. + +For the locate key, the component.csv file is used for province code lookup +The province.csv and component.csv files both contain province data. Province.csv has the numeric geonames province code. Component.csv has the province name abbreviation. + +For the nominatim key, the category.csv file is used to provide French translation for the category field \ No newline at end of file diff --git a/docs/lambda/geolocator/geolocator-bucket-content/tables/category.csv b/docs/lambda/geolocator/geolocator-bucket-content/tables/category.csv new file mode 100644 index 00000000..cd8c76e5 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/tables/category.csv @@ -0,0 +1,23 @@ +code,en,fr +aerialway,Aerialway,Voie aérienne +aeroway,Aeroway,Aéroparc +amenity,Amenity,Commodité +boundary,Boundary,Limite +building,Building,Bâtiment +city,City,Ville +highway,Highway,Autoroute +intersection,Intersection,Intersection +landuse,Land use,Utilisation des terres +leisure,Leisure,Loisirs +man_made,Man made,Élément anthropique +natural,Natural,Naturel +nts,NTS,NTS +office,Office,Bureau +place,Place,Lieu +postalcode,Postal Code,Code postal +shop,Shop,Magasin +street,Street,Rue +tourism,Tourism,Tourisme +town,Town,Ville +village,Village,Village +waterway,Waterway,Voie navigable diff --git a/docs/lambda/geolocator/geolocator-bucket-content/tables/component.csv b/docs/lambda/geolocator/geolocator-bucket-content/tables/component.csv new file mode 100644 index 00000000..21bddb35 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/tables/component.csv @@ -0,0 +1,17 @@ +Code,en,fr +NL,Newfoundland and Labrador,Terre-Neuve-et-Labrador +PE,Prince Edward Island,Île-du-Prince-Édouard +NS,Nova Scotia,Nouvelle-Écosse +NB,New Brunswick,Nouveau-Brunswick +QC,Quebec,Québec +ON,Ontario,Ontario +MB,Manitoba,Manitoba +SK,Saskatchewan,Saskatchewan +AB,Alberta,Alberta +BC,British Columbia,Colombie-Britannique +YU,Yukon,Yukon +YT,Yukon,Yukon +NT,Northwest Territories,Territoires du Nord-Ouest +NU,Nunavut,Nunavut +UF,Undersea Feature,Entité sous-marine +IW,International Waters,Eaux internationales diff --git a/docs/lambda/geolocator/geolocator-bucket-content/tables/generic.csv b/docs/lambda/geolocator/geolocator-bucket-content/tables/generic.csv new file mode 100644 index 00000000..ff3f6831 --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/tables/generic.csv @@ -0,0 +1,1248 @@ +Code,en,fr +1,City,Cité +2,Town,Ville +3,Village,Village +4,Metropolitan Municipality,Municipalité métropolitaine +6,Community,Agglomération +9,Separated Town,Ville séparée +11,Hamlet,Hameau constitué +12,Summer Village,Village estival +14,Municipality,Municipalité +15,City,Cité +16,Resort Municipality,Municipalité de villégiature +17,Organized Hamlet,Hameau organisé +18,Resort Village,Village-villégiature +19,Northern Hamlet,Hameau nordique +20,Northern Settlement,Établissement nordique +21,Northern Village,Village nordique +22,Municipality,Municipalité +23,Town,Ville +24,Urban Community,Agglomération urbaine +25,Metropolitan Corporation,Corporation métropolitaine +26,Charter Community,Communauté à chartre +27,Township Municipality,Municipalité de canton +28,United Townships Municipality,Municipalité de cantons unis +29,Parish Municipality,Municipalité de paroisse +30,Village Municipality,Municipalité de village +31,Cree Village Municipality,Municipalité de village cri +33,Naskapi Village Municipality,Municipalité de village naskapi +34,Northern Village Municipality,Municipalité de village nordique +35,District Municipality,Municipalité de district +36,Community Government,Gouvernement communautaire +37,Community,Communauté +38,Borough,Arrondissement +39,Mountain Resort Municipality,Municipalité de villégiature de montagne +40,Rural Community,Communauté rurale +41,Upper-tier Municipality,Municipalité de palier supérieur +42,Lower-tier Municipality,Municipalité de palier inférieur +43,Single-tier Municipality,Municipalité à palier unique +44,Northern Town,Ville nordique +100,Hamlet,Hameau +101,Village,Village +102,Police Village,Village partiellement autonome +103,Rural Village,Village rurale +104,Settlement,Établissement +105,Compact Rural Community,Collectivité rurale concentrée +107,Resort,Centre de villégiature +108,Locality,Localité +109,Named Locality,Lieu-dit +110,Railway Junction,Jonction ferroviaire +111,Railway Point,Point ferroviaire +112,Station,Gare +113,Post Office,Bureau de poste +114,Landing,Débarcadère +115,Siding,Voie d'évitement +116,Depot,Dépôt +117,Abandoned Locality,Localité abandonnée +119,Trading Post,Poste de traite +120,Fort,Fort +121,Community,Communauté +122,Dispersed Rural Community,Collectivité rurale dispersée +123,Hamlet,Hameau +124,Locality,Localité +125,Post Office,Bureau de poste +126,Town,Ville +127,Railway Junction,Jonction +129,Station,Gare +130,Military Post Office,Bureau de poste militaire +131,Resort,Centre de villégiature +132,Weather Station,Station météorologique +133,Hutterite Colony,Communauté agricole huttérite +137,Colony,Communauté agricole +139,Trailer Park,Parc de maisons mobiles +141,Customs Point,Poste de douane +143,Summer Post Office,Bureau de poste estival +144,Flag Stop,Arrêt ferroviaire +145,Metis Settlement,Établissement métis +146,Native Community,Communauté autochtone +147,Indian Settlement,Établissement amérindien +148,Northern Community,Communauté nordique +149,Unincorporated Village District,District du village non constitué en corporation +151,Ski Area,Centre de ski +152,Ranch,Ranch +154,Vacated or Seasonal Settlement,Établissement abandonné ou saisonnier +155,Cannery,Conserverie +156,Indian Village,Village indien +157,Former First Nation Village,Ancien village des premières nations +158,Former Locality,Ancienne localité +159,Former Post Office,Ancien bureau de poste +160,Forest Village,Village forestier +161,Naskapi Village,Village naskapi +162,Station,Station de chemin de fer +163,Amerindian Settlement,Établissement amérindien +164,Hotel,Établissement hôtelier +167,Trailer Park,Parc de maisons mobiles +168,Trading Post,Poste de traite +170,Station,Gare ferroviaire +171,Customs Point,Bureau de douane +172,Railway Stop,Arrêt ferroviaire +173,Neighbourhood,Village +174,Village,Village +175,Cree Village,Village cri +176,Railway Yard,Gare de triage +177,Customs Point,Poste de douane +178,Weather Station,Station météorologique +179,Research Station,Station de recherche +180,Former Railway Point,Ancien point ferroviaire +181,Settlement Corporation,Établissement corporatif +183,Former Cannery,Ancienne conserverie +184,Townsite,Lotissement urbain +185,Woodland Area,Région boisée +186,First Nation Village,Village des premières nations +187,Recreational Community,Communauté récréationnelle +200,Urban Community,Agglomération urbaine +201,Neighbourhood,Quartier +202,Suburban Community,Collectivité suburbaine +203,Industrial Park,Parc industriel +204,Industrial Area,Zone industrielle +205,Residential Area,Secteur résidentiel +206,Subdivision,Subdivision +207,Industrial Park,Parc industriel +208,Metropolitan Area,Région métropolitaine +209,Local Urban District,District urbain local +212,Residential Area,Ensemble résidentiel +213,Neighbourhood,Quartier +214,Administrative Sector,Secteur +400,Rural District,District rural +401,Local Government District,District de gouvernement local +402,Local Service District,District de service local +403,Local District Municipality,Municipalité de district local +404,Municipal District,District municipal +405,Indian Government District,District sous administration indienne +406,Township Municipality,Municipalité de canton +407,District Municipality,Municipalité de district +408,Regional Municipality,Municipalité régionale +409,Rural Municipality,Municipalité rurale +410,Regional District,District régional +411,County,Comté +412,Region,Région +413,Health District,District de santé +414,School District,District d'école +415,Improvement District,District d'urbanisme +416,Electoral District,Circonscription électorale +417,Local Improvement District,District d'amélioration locale +418,Indian Government District: Land Unit,District sous administration indienne : unité territoriale +419,County Municipality,Municipalité de comté +420,Restructured County,Comté restructuré +421,Special Area,Zone spéciale +422,Municipal Electoral District,Circonscription électorale municipale +423,Provincial Electoral District,Circonscription électorale provinciale +424,Regional Municipality,Communauté régionale +425,Regional Municipality,Communauté urbaine +426,County Regional Municipality,Municipalité régionale de comté +427,Unorganized Territory,Territoire non organisé +428,Judicial District,District judiciaire +430,Municipal County,Comté municipal +434,Local Services Area,Région de services locaux +435,Local Development Area,Zone locale d'aménagement +437,Administrative Region (1966),Région administrative (1966) +438,Administrative Region (1985),Région administrative (1985) +439,Administrative Region,Région administrative +440,Township Municipalities,Municipalités de canton +441,Specialized Municipality,Municipalité spécialisée +442,Territory equivalent to an MRC,Territoire équivalent à une MRC +445,Grant Lot,Arrière-fief +446,Grant,Concession +447,Registration Area,Division d'enregistrement +450,Seigniory,Seigneurie +451,Grant,Fief +452,Grant Lot Subdivision,Arrière-arrière-fief +453,Cadastral Division,Division cadastrale +480,Country,Pays +481,County,Comté +482,Land District,District territorial +483,Division,Division +484,Parish,Lot riverain +485,Geographic Township,Canton géographique +486,Geographic District,District géographique +487,Regional District,District régional +488,District,District +489,Parish,Paroisse +490,Land Grant,Concession +491,Province,Province +492,Territory,Territoire +493,Region,Région +495,Township,Canton +496,Concession,Rang +497,County,Comté +498,Parish,Paroisse +499,Lot,Lot +500,Royalty,Royauté +501,Census District,Division de recensement +502,Settlement,Établissement +504,Census Division,Division de recensement +505,Administrative Region,Territoire +506,Geographic County,Comté géographique +507,Gores,Abouts +508,Region,Région +510,Territory,Territoire +511,Township,Canton +512,Indian Reserve,Réserve indienne +513,Administrative Region,Territoire +514,Lots,Lots +515,"Geographic Township, Range","Canton géographique, Rang" +516,Migratory Bird Sanctuary,Refuge d'oiseaux migrateurs +517,National Wildlife Area,Réserve nationale de faune +518,Provincial Heritage Property,Propriété du patrimoine provinciale +519,Biosphere Reserve,Réserve de la biosphère +520,Wildlife Refuge,Refuge faunique +521,Managed Hunting Area,Zone d'exploitation contrôlée +522,Conservation Reserve,Zone d'aménagement et de conservation +523,National Marine Conservation Area,Aire marine nationale de conservation +524,Conservation Area,Réserve de pêche +525,Conservation Area,Réserve de chasse +526,National Marine Conservation Area,Aire marine nationale de conservation +527,Protected Area,Zone protégée +528,Experimental Farm,Ferme expérimentale +529,Provincial Natural Park,Parc naturel provincial +530,Provincial Wilderness Park,Parc naturel provincial +531,Provincial Wayside Park,Halte provinciale +532,World Heritage Site,Site du patrimoine mondial +533,National Marine Conservation Area Reserve,Réserve d'aire marine nationale de conservation +534,National Marine Conservation Area Reserve,Réserve d'aire marine nationale de conservation +535,World Heritage Site,Site du patrimoine mondial +536,National Marine Park,Parc marin national +537,National Marine Park,Parc marin national +538,National Historic Site,Lieu historique national +539,National Historic Site,Lieu historique national +540,Territorial Park,Parc territorial +541,National Wildlife Area,Réserve nationale de faune +542,Game Sanctuary,Refuge faunique +544,National Park,Parc national +545,Provincial Park,Parc provincial +546,Provincial Game Sanctuary,Réserve faunique provinciale +547,Recreation Area,Aire récréative +548,Provincial Heritage Park,Parc provincial du patrimoine +549,Wildlife Management Area,Région d'aménagement faunique +550,Public Park,Parc public +551,Hunting and Fishing Reserve,Réserve de chasse et pêche +553,Provincial Park Reserve,Réserve de parc provinciale +554,Bird Sanctuary,Refuge d'oiseaux +555,Conservation Area,Zone de conservation +556,Game Preserve,Réserve faunique +557,Provincial Forest,Forêt provinciale +558,Camp Site,Base de plein air +559,Conservation Park,Parc de conservation +560,Recreation Park,Parc de récréation +561,Migratory Bird Sanctuary,Refuge d'oiseaux migrateurs +562,Park,Parc de récréation +563,Refuge,Refuge +564,Historic Site,Site historique +565,Provincial Recreation Park,Parc provincial de loisirs +566,Forest Reserve,Réserve forestière +567,Provincial Game Sanctuary - Waterfowl,Réserve faunique provinciale - Gibier d'eau +568,Provincial Wildlife Management Area,Zone de gestion de faune du gouvernement provincial +569,Managed Hunting Area,Région de chasse contrôlée +570,Provincial Historic Village,Village historique provincial +571,Provincial Historic Village,Village historique provincial +572,Wildlife Refuge,Refuge faunique +573,Wildlife Sanctuary,Refuge faunique +574,Historic Site,Site historique +575,Municipal Park,Parc municipal +576,Conservation Reserve,Zone de conservation +577,Picnic Area,Aire de pique-nique +578,Protected Area,Aire protégée +579,Provincial Nature Reserve,Réserve naturelle provinciale +580,International Park,Parc international +581,Game Refuge,Refuge faunique +582,International Park,Parc international +583,Regional Park,Parc régional +584,Ecological Reserve,Réserve écologique +585,Provincial Historic Park,Parc historique provincial +586,National Park,Parc national +587,Ecological Reserve,Réserve écologique +589,Zoological Garden,Jardin zoologique +590,Wildlife Reserve,Réserve faunique +591,Recreation Site,Site récréatif +592,Provincial Historic Site,Lieu historique provincial +593,Wildlife Management Unit,Unité d'aménagement faunique +594,Provincial Marine Park,Parc maritime provincial +595,Provincial Historic Site,Lieu historique provincial +596,Wilderness Area,Réservation site +597,National Park Reserve,Réserve de parc national +598,National Park Reserve,Réserve de parc national +599,Provincial Recreation Area,Région de récréation provinciale +600,River,Rivière +601,Brook,Ruisseau +602,Creek,Crique +603,Outlet,Décharge +604,Pup,Crique +605,Drain,Rigolet +606,River,Fleuve +607,Canal,Canal +610,Branch,Branche +612,Gully,Ruisseau +613,Millstream / Mill Stream,Ruisseau de moulin +614,Stream,Ruisseau +615,Run,Ruisseau +616,Runoff,Ruisseau +617,Stream,Ruisseau +621,River,Rivière +622,Drain,Rigolet +623,Branch,Branche +624,Brook,Ruisseau +626,Outlet,Décharge +627,Dribble,Crique +628,Feeder,Ruisseau +629,Fork,Fourche +630,Creek,Crique +631,Inlet,Ruisseau +632,Ditch,Ruisseau +634,Gulch,Coulée +635,Fork,Fourche +636,Coulée,Coulée +637,Overflow,Drain +638,Ditches,Ruisseaux +639,Canals,Canaux +640,Channel,Chenal +641,Rivulet,Crique +643,Creeks,Criques +645,Runs,Rapides +646,Gullies,Criques +647,Rivers,Rivières +648,Brooks,Ruisseaux +649,Rivers,Rivières +650,Coulee,Crique +651,River Segment,Segment de rivière +652,Drain,Rigole +653,Gut,Ruisseau +654,River segment,Segment de rivière +700,Bend,Courbe +701,River Junction,Confluent +703,Forks,Confluent +704,Reach,Passe +705,Oxbow,Méandre +706,River Mouth,Embouchure +708,Snye,Faux chenal +710,Pool,Fosse +711,Deadwater,Eau morte +712,Stillwater,Eau morte +713,Bogan,Bras mort +714,Cove,Bras mort +715,Crossing,Traverse +716,Elbow,Coude +717,Hole,Fosse +719,Pothole,Eau morte +722,Padou,Cove +724,Runround,Faux chenal +725,Turn,Courbe +726,Turns,Courbes +727,Fork,Confluent +728,Confluence,Confluent +729,Whirlpool,Remous +730,Rapids,Rapides +731,Falls,Chutes +732,Cascade,Cascade +733,Cascades,Cascades +734,Fall,Chute +735,Rapid,Rapide +736,Waterfall,Chute +737,Dump,Chute +740,Guzzle,Ruisseau +742,Rip,Rapide +743,Rips,Rapides +744,Falls,Sault +746,Current,Courant +747,Channel,Branche +748,Ford,Gué +749,Slough,Faux chenal +750,Rapids,Rapides +751,Rattle,Rapide +752,Cataract,Cataracte +753,Current,Courant +755,Crossing,Traverse +757,Salmon Hole,Fosse à saumon +758,Riffle,Rapide +759,Fork,Confluent +760,Bend,Courbe +761,Meander,Méandre +762,Rapid,Rapide +763,Eddy,Remous +765,Horseshoe,Coude +766,Falls,Chutes +768,Waterfall,Chute +769,Basin,Bassin +770,Stillwaters,Eaux mortes +771,Deadwaters,Eaux mortes +772,Pools,Fosses +773,Run,Rapide +774,Runs,Rapides +775,Fishing Pool,Lieu de pêche +776,Fishing Pools,Lieux de pêche +777,Fishing Hole,Lieu de pêche +778,Whirlpools,Remous +779,Bends,Courbes +781,Shallows,Hauts-fonds +782,Flumes,Rapides +783,Stretch,Passe +787,Meanders,Méandres +793,Elbow,Coude +795,Drain,Cours d'eau agricole +796,Fishing Hole,Lieu de pêche +797,Fishing Holes,Lieux de pêche +798,Inlet,Embouchure +799,Mouth,Embouchure +951,Lake,Lac +952,Lakes,Lacs +953,Pond,Lac +954,Ponds,Lacs +955,Reservoir,Réservoir +956,Slough,Marécage +957,Lagoon,Lagune +958,Barasway,Barasway +959,Flowage,Réservoir +960,Hole,Étang +961,Lake,Lac +962,Pond,Lac +963,Millpond / Mill Pond,Réservoir +964,Pool,Étang +965,Pothole,Fosse +966,Pughole / Pug Hole,Étang +967,Mal Bay,Barachois +968,Pond,Mare +969,Sloughs,Marécages +970,Lake,Lac +971,Gully,Marais +972,Lakes,Lacs +973,Lagoon,Lagune +974,Pond,Étang +975,Barachois,Barachois +976,Gullies,Marais +977,Potholes,Étangs +979,Lake,Lac +980,Basin,Lac +981,Holes,Fosse +982,Pond,Lac +983,Ponds,Lacs +984,Pools,Étangs +985,Ponds,Étangs +986,Steady,Fosse +987,Steadies,Fosses +988,Ponds,Mares +989,Waters,Étangs +990,Tarn,Lac +991,Tarns,Lacs +992,Artificial Lake,Lac artificiel +993,Artificial Lakes,Lacs artificiels +994,Part of a Lake,Partie de lac +995,Part of a Reservoir,Partie de réservoir +996,Reservoir,Réservoir +997,Aboiteau,Aboiteau +998,Part of a Lake,Partie de lac +1150,Spring,Source +1151,Springs,Sources +1152,Hotspring / Hot Spring,Source thermale +1153,Hotsprings / Hot Springs,Sources thermales +1154,Spring,Source +1158,Springs,Sources +1200,Canal,Chenal +1201,Channel,Chenal +1202,Channels,Chenaux +1203,Narrows,Passage +1204,Strait,Détroit +1205,Sound,Détroit +1206,Pass,Passage +1207,Passage,Passage +1208,Tickle,Passage +1209,Opening,Passage +1210,Gut,Passage +1211,Straits,Détroits +1212,Ocean,Océan +1213,Sea,Mer +1214,Chokey,Passe +1215,Dugway,Passage +1216,Entrance,Entrée +1217,Gap,Passage +1218,Gulch,Goulet +1220,Guzzle,Goulet +1221,Hole,Passage +1222,Inlet,Passage +1224,Run,Passage +1225,Sluice,Passage +1227,Way,Passage +1228,Thoroughfare,Chenal +1229,Cut,Canal artificiel +1230,Pass,Passe +1231,Channel,Chenal +1233,Strait,Détroit +1235,Traverse,Passe +1236,Creek,Passage +1237,Corners,Carrefour +1239,Runs,Passage +1240,Basin,Bassin +1242,Basin,Bassin +1243,Gully,Goulet +1244,Gully,Goulet +1246,Pass,Passes +1248,Sea,Mer +1249,Channels,Chenaux +1250,Ocean,Océan +1352,Rip,Ride +1353,Rips,Rides +1354,Tidal Flow,Courant de marée +1355,Tickles,Passage +1356,Tittle,Passage +1358,Tidal Rapids,Courant de marée +1400,Bay,Baie +1401,Road,Rade +1402,Arm,Bras +1403,Fiord,Fjord +1404,Gulf,Golfe +1405,Harbour,Havre +1406,Anchorage,Mouillage +1407,Cove,Anse +1408,Coves,Anses +1409,Motion,Ride +1410,Creek,Crique +1411,Inlet,Bras +1412,Bight,Baie +1413,Hole,Trou +1417,Creek,Crique +1418,Eddy,Remous +1419,Estuary,Estuaire +1421,Goulet,Goulet +1422,Gully,Anse +1423,Haven,Havre +1424,Hole,Trou +1426,Port,Mouillage +1427,Soi,Barachois +1428,Sound,Bras +1429,Lead,Passage +1431,Creek,Ruisseau +1432,Cove,Anse +1433,Bay,Baie +1434,Fiord,Fjord +1435,Harbour,Havre +1436,Anchorage,Rade +1437,Basin,Bassin +1438,Anchorage,Mouillage +1439,Pool,Fosse +1440,Bite,Baie +1441,Arm,Bras +1444,Oxbow,Bras mort +1445,Estuary,Estuaire +1446,Gulf,Golfe +1448,Bays,Baies +1449,Harbours,Havres +1450,Holes,Anses +1451,Roads,Rades +1452,Gullies,Anses +1453,Basin,Anse +1455,Gulch,Anse +1456,Gulches,Anses +1457,Coves,Anses +1458,Bays,Baies +1459,Basins,Bassins +1460,Bottom,Fond +1461,Oxbows,Bras morts +1463,Backside,Anse +1464,Gut,Anse +1465,Part of a bay,Partie de baie +1600,Cape,Cap +1601,Head,Cap +1602,Isthmus,Isthme +1603,Peninsula,Péninsule +1604,Point,Pointe +1605,Promontory,Promontoire +1606,Headland,Cap +1607,Foreland,Pointe +1608,Spit,Flèche +1609,Points,Pointes +1610,Heights,Hauteurs +1611,Land,Terre +1612,Coast,Côte +1613,Arch,Arche +1614,Bank,Berge +1615,Cliff,Falaise +1616,Coast,Côte +1618,End,Pointe +1619,Hummock,Buttereau +1620,Neck,Langue de terre +1621,Nose,Pointe +1622,Sand Point,Pointe de sable +1623,Seaside,Rivage +1624,Head,Tête +1625,Bluff,Falaise +1626,Bill,Pointe +1627,Point,Pointe +1628,Peninsula,Presqu'île +1629,Cap,Cap +1630,Capes,Caps +1631,Peninsula,Péninsule +1632,Bend,Pointe +1633,Cliff,Falaise +1639,Delta,Delta +1640,Promontory,Promontoire +1641,Cliffs,Falaises +1642,Capes,Caps +1643,Bluffs,Falaises +1644,Heads,Caps +1645,Hummocks,Collines +1646,Cliffs,Falaises +1647,Points,Pointes +1648,Spit,Langue de terre +1649,Arches,Arches +1650,Coasts/ Shores,Côtes +1651,Blow-Me-Down / Blow Me Down,Promontoire +1652,Bank,Berge +1653,Part of a coastline,Segment de côte +1900,Beach,Plage +1901,Bar,Barre +1902,Breaker,Brisant +1903,Breakers,Brisants +1904,Sand Bar,Batture +1905,Sand Bar,Batture +1906,Dune,Dune +1907,Flat,Batture +1908,Flats,Battures +1909,Shore,Rivage +1910,Beach,Plage +1912,Strand,Estran +1913,Beach,Grève +1915,Sandspit,Flèche +1916,Bar,Barre +1918,Spur,Éperon +1919,Gradins de plage,Gradins de plage +1920,Shore,Rivage +1922,Beaches,Plages +1923,Bars,Barres +1924,Flats,Battures +1926,Beaches,Grèves +1930,Spit,Flèche +1931,Strand,Rivage +1932,Strands,Rivages +1933,Raised Beach,Plage soulevée +1934,Tidal Flat,Replat de marée +2000,Bank,Banc +2001,Banks,Bancs +2002,Reef,Récif +2003,Reefs,Récifs +2004,Shoals,Hauts-fonds +2005,Shoal,Haut-fond +2006,Ledge,Chaussée +2008,Trough,Cuvette +2009,Plain,Plaine +2010,Bar,Barre +2013,Shoal or Patch,Bosse +2014,Chuckle,Rochers +2015,Ground,Haut-fond +2016,Grounds,Hauts-fonds +2019,Knoll,Dôme +2021,Patch,Haut-fond +2022,Peak,Pic +2023,Ridge,Dorsale +2024,Rock,Rocher +2025,Spot,Haut-fond +2026,Sunker,Rocher +2027,Sand Bank / Sandbank,Berge +2028,Bottom,Bottom +2029,Canyon,Canyon +2030,Breakers,Brisants +2031,Shoal,Haut-fond +2032,Rock,Roche +2033,Reefs,Récifs +2034,Ground,Banc de pêche +2035,Sand Bank,Banc de sable +2036,Reef,Récif +2037,Range,Chaîne +2038,Grounds,Bancs de pêche +2039,Bank,Banc +2041,Back,Haut-fond +2042,Bantam,Haut-fond +2043,Deeps,Bas-fond +2045,Hook,Poulier +2046,Bars,Barres +2047,Inundated Area,Marécage +2048,Ledges,Chaussées +2049,Sand Banks / Sandbanks,Bancs de sable +2050,Banks,Bancs +2051,Channel,Vallée +2052,Valley,Vallée +2053,Sunkers,Rochers +2054,Patches,Hauts-fonds +2055,Shoals,Hauts-fonds +2056,Banks,Bancs +2057,Sand Bar,Cordon littoral +2058,Sand Banks,Bancs de sable +2059,Bank,Banc +2061,Rock,Écueil +2062,Fosse,Fosse +2063,Shelf,Plate-forme insulaire +2065,Rise,Massif +2066,Basin,Bassin +2067,Hummock,Buttereau +2068,Lump,Bosse +2069,Rocks,Roches +2300,Island,Île +2301,Islands,Îles +2302,Isle,Île +2303,Isles,Îles +2306,Archipelago,Archipel +2307,Islet,Îlot +2308,Islets,Ilots +2312,Islet,Cap +2313,Cay,Caye +2315,Nubble,Rocher +2316,Thrum,Rocher +2317,Thrumcap / Thrum Cap,Rocher +2319,Group,Groupe +2320,Islet,Îlette / Ilette +2321,Mound,Îlot +2322,Island,Île +2323,Islands,Îles +2324,Islet,Îlot +2325,Archipelago,Archipel +2428,Rock,Rocher +2429,Rock Pile,Îlot +2430,Rocks,Roches +2431,Crab,Îlot +2432,Thrums,Rochers +2433,Rocks,Rochers +2434,Islets,Îlots +2435,Cays,Cayes +2436,Brandies,Rochers +2700,Range,Chaîne de montagnes +2701,Mountain,Montagne +2702,Mountains,Montagnes +2703,Peak,Pic +2704,Peaks,Pics +2705,Hill,Colline +2706,Hills,Collines +2707,Escarpment,Escarpement +2708,Cliff,Escarpement +2709,Cliffs,Escarpements +2710,Ridge,Chaînon +2711,Esker,Esker +2712,Bluff,Falaise +2713,Slope,Pente +2714,Dune,Dune +2715,Summit,Sommet +2716,Butte,Butte +2717,Highland,Massif +2718,Point,Pic +2719,Ramparts,Falaise +2720,Ranges,Chaînes de montagnes +2721,Bluffs,Falaises +2722,Pingo,Pingo +2723,Ridges,Chaînons +2724,Buttes,Buttes +2725,Hill,Côte +2726,Highlands,Massifs +2727,Shoulder,Chaînon +2729,Drumlin,Colline +2731,Pinnacle,Pinacle +2732,Pinnacles,Pinacles +2733,Physiographic Region,Région physiographique +2735,Crag,Pic +2739,Knoll,Colline +2740,Ledge,Corniche +2741,Lookoff,Belvédère +2742,Mount,Mont +2743,Mount,Mont +2744,Hills,Collines +2745,Scarp,Escarpement +2746,Spur,Éperon +2749,Lookout,Belvédère +2750,Peak,Pointu +2751,Rim,Bordure +2752,Rock,Falaise +2753,Ridge,Arête +2755,Foothills,Piémont +2756,Group,Groupe +2757,Rock,Roche +2758,Dome,Dôme +2759,Spire,Flèche +2760,Tower,Pic +2763,Hill,Colline +2766,Mountain,Montagne +2767,Rock,Falaise +2770,Back,Colline +2774,Brow,Sommet +2775,Buse,Colline +2777,Cone,Colline +2789,Ledge,Corniche +2790,Ridge,Crête +2791,Dome,Dôme +2793,Foothills,Piémont +2795,Summit,Sommet +2798,Hump,Bosse +2801,Bank,Côte +2803,Sand Hills,Dunes +2805,Moraine,Moraine +2806,Landslide,Glissement de terrain +2807,Knob,Colline +2809,Summits,Sommets +2810,Domes,Dômes +2812,Peak,Piton +2813,Mounts,Monts +2814,Mountains,Montagnes +2815,Crags,Pics +2816,Ledges,Corniches +2817,Hills,Côtes +2818,Dunes,Dunes +2819,Hills,Coteaux +2820,Highlands,Massif +2821,Tolt,Buttereau +2822,Range,Chaîne +2823,Spires,Flèches +2826,Towers,Pics +2827,Wall,Muraille +2828,Knolls,Buttereaux +2829,Abrupt,Abrupt +2830,Arch,Pont naturel +2831,Lookout,Belvédère +2832,Bank,Versant +2833,Summits,Sommets +2834,Abrupts,Abrupts +2835,Head,Sommet +2836,Hill,Coteau +2837,Tolts,Buttereaux +2838,Buffalo Jump,Précipice à bisons +2839,Lookouts,Belvédères +2840,Rocks,Rochers +2843,Slide,Glissement +2844,Cairn,Colline +2845,Cairns,Cairns +2847,Escarpment,Escarpement +2849,Eskers,Eskers +2850,Cliff,Paroi +2851,Peak,Pic +2854,Range,Chaîne de montagnes +2855,Ridges,Crêtes +2856,Fault,Faille +2859,Monument,Monument +2860,Cuesta,Cuesta +2901,Ravine,Ravin +2902,Coulée,Coulée +2903,Valley,Vallée +2904,Gulch,Ravin +2906,Canyon,Canyon +2907,Gorge,Gorge +2908,Gully,Ravin +2909,Gorges,Gorges +2911,Glen,Vallée +2912,Hollow,Vallon +2913,Pass,Col +2914,Vault,Ravin +2915,Notch,Col +2916,Basin,Cirque +2917,Depression,Dépression +2918,Gap,Col +2919,Trench,Fossé +2921,Saddle,Col +2922,Ravine,Ravin +2923,Valley,Vallée +2924,Niche,Niche +2925,Col,Col +2926,Vale,Vallée +2928,Cirque,Cirque +2929,Crater,Cratère +2930,Crevasse,Crevasse +2932,Draw,Ravine +2934,Cul-de-sac,Cul-de-sac +2935,Défilé,Défilé +2936,Depression,Dépression +2937,Crater,Cratère +2938,Crater,Cratère météorique +2939,Hollow,Dépression +2940,Canon,Canyon +2941,Cirques,Cirques +2942,Pup,Vallon +2943,Coulee,Coulée +2944,Hole,Hole +2950,Valley,Vallon +2954,Drook,Ravine +2955,Depression,Kettle +2956,Cleft,Fissure +2957,Cañon,Canyon +3100,Plateau,Plateau +3101,Plain,Plaine +3102,Plains,Plaines +3104,Lowland,Plaine +3105,Flat,Plaine +3106,Flats,Plaines +3107,Upland,Haute-terre +3108,Uplands,Hautes-terres +3109,Terrace,Terrasse +3111,Bench,Terrasse +3112,Plain,Plaine +3113,Terrace,Terrasse +3114,Plains,Plaines +3115,Crossing Place,Passage +3200,Glacier,Glacier +3201,Icefield,Champ de glace +3202,Ice Cap / Icecap,Calotte glaciaire +3203,Névé,Névé +3204,Ice Shelf,Plate-forme de glace +3205,Glaciers,Glaciers +3206,Snowfield,Champ de neige +3207,Ice Moraine,Moraine de glace +3208,Ice Rise,Sommet de glace +3209,Ice Sheet,Glacier +3211,Glacieret,Petit Glacier +3213,Icefall,Chute de glace +3214,Ice Patch,Parcelle de glace +3500,Forest,Forêt +3501,Bluff,Terrain boisé +3502,Desert,Terrain boisé +3503,Wooded Area,Terrain boisé +3504,Woods,Bois +3505,Grove,Bosquet +3506,Experimental Forest,Forêt expérimentale +3507,Grove,Bosquet +3508,Woods,Bois +3509,Forest,Forêt +3510,Bush,Bosquet +3511,Forest Area,Forêt expérimentale +3512,Tree Nursery,Pépinière +3515,Cedar Grove,Cédrière +3516,Orchard,Verger +3570,Meadow,Baissière +3571,Meadows,Baissières +3572,Barren,Pré +3573,Barrens,Pré +3574,Moor,Pré +3575,Haymarsh,Baissière +3576,Hay Meadow,Baissière +3577,Meadow,Pré +3578,Savannah,Baissière +3579,Prairie,Prairie +3580,Marsh,Marche +3581,Marshes,Marches +3582,Swamp,Marécage +3583,Bog,Marais +3585,Fen,Marais +3586,Flat,Baissière +3587,Flats,Baissières +3588,Glade,Baissière +3589,Heath,Pré +3590,Intervale,Platin +3592,Slough,Baissière +3593,Swale,Baissière +3594,Bog,Mocauque +3595,Plain,Marécage +3596,Bog,Marais +3597,Peat Bog,Tourbière +3602,Quagmire,Fondrière +3603,Swamp,Marécage +3605,Bog,Savane +3606,Community Pasture,Commune +3607,Muskeg,Fondrière +3610,Fallow,Clairière +3611,Plains,Baissière +3612,Swales,Baissière +3613,Bogs,Marais +3614,Glades,Baissières +3615,Hole,Baissière +3617,Intervale,Platin +3618,Bogs,Savanes +3619,Field,Champ +3620,Barrens,Prés +3621,Blueberry Grove,Bleuetière +3622,Fields,Champs +3623,Marshes,Marécages +3624,Bottom,Fond +3626,Meadow,Alpage +3628,Sloughs,Marécages +3700,Cave,Caverne +3701,Caves,Cavernes +3702,Cave,Caverne +3703,Grotto,Grotte +3705,Cave,Antre +3900,Fumarole,Fumerolle +3901,Volcano,Volcan +3902,Crater,Cratère +3903,Crater,Cratère +3904,Volcanoes,Volcans +4101,Generating Station,Centrale hydroélectrique +4102,Oilwell,Puit de pétrole +4103,Mine,Mine +4104,Mines,Mines +4105,Drill Site,Lieu de forage +4106,Quarry,Carrière +4107,Mining Camp,Camp minier +4109,Pits,Carrières +4110,Quarry,Carrière +4111,Forest Camp,Camp forestier +4112,Forest Locality,Domaine forestier +4114,Pit,Carrière +4117,Generating Station,Centrale hydroélectrique +4119,Forestry Station,Station forestière +4120,Quarries,Carrières +4121,Forest Depot,Dépôt forestier +4122,Forest Education Centre,Centre éducatif forestier +4123,Mining Camp,Camp minier +4301,Portage,Portage +4302,Road,Chemin +4303,Trail,Sentier +4305,Ferry,Traverse +4307,Bridge,Pont +4308,Causeway,Chaussée +4309,Waterway,Voie navigable +4310,Waterway,Voie navigable +4311,Lock,Écluse +4312,Locks,Écluses +4313,Harbour,Havre +4314,Airport,Aéroport +4315,Breakwater,Digue +4316,Canal,Chenal +4317,Landing,Débarcadère +4318,Port,Débarcadère +4319,Railway Yard,Gare de triage +4320,Lock,Écluse +4321,Locks,Écluses +4322,Dock,Quai +4323,Airport,Aéroport +4324,Seaplane Base,Base d'hydravions +4325,Canal,Canal +4326,Dock,Quai +4327,Wharf,Débarcadère +4328,Marina,Port de plaisance +4329,Highway,Autoroute +4330,Route,Autoroute +4331,Steamer Landing,Débarcadère +4333,Airport,Aérodrome +4334,Flag Stop,Arrêt +4335,Harbour,Bassin portuaire +4336,Bridge,Pont +4337,Landing,Débarcadère +4338,Route,Portage +4339,Airfield,Aérodrome +4341,Path,Sentier +4342,Portages,Portages +4343,Airstrip,Piste d'atterrissage +4345,Corner,Jonction +4348,Terminal,Quai +4349,Road Bend,Courbe routière +4351,Deepwater Terminal,Terminal en eau profonde +4352,Seaway,Voie maritime +4353,Seaway,Voie maritime +4357,Piers,Débarcadères +4358,Railway Cut,Coupe de chemin de fer +4361,Road,Chemin +4363,Highway,Autoroute +4364,Pier,Débarcadère +4365,Trail,Piste +4368,Railway Junction,Embranchement ferroviaire +4370,Côte,Côte +4371,Covered Bridge,Pont couvert +4372,Bridges,Ponts +4373,Tunnel,Tunnel +4374,Seaplane Base,Hydrobase +4376,Marina,Port de plaisance +4379,Bridge and Tunnel,Pont-tunnel +4380,Trail,Sentier +4381,Lighthouse,Phare +4385,Tunnels,Tunnels +4386,Railway,Chemin de fer +4387,Historical Route,Sentier historique +4388,Junction,Jonction +4389,Route,Itinéraire +4390,Forest Resources Road,Chemin des ressources forestières +4391,Scenic Touring Route,Route panoramique touristique +4501,Research Station,Station de recherche +4502,Amusement Park,Parc d'amusement +4503,Cabin,Cabane +4504,Camp,Camp +4506,Canadian Forces Base,Base des Forces canadiennes +4507,Dam,Barrage +4508,Cairn,Cairn +4509,Wreck,Épave +4510,Drain,Canal artificiel +4511,Dyke,Digue +4514,Seasonal Camp,Camp saisonnier +4515,Dam,Barrage +4516,Ditch,Fossé +4517,Sports Camp,Camp sportif +4518,Military Base,Base militaire +4519,Dyke,Fossé +4520,Dam,Digue +4521,Ditch,Fossé +4522,Diversion,Canal de dérivation +4523,Climate Station,Station climatique +4525,Farm,Ferme +4526,Fishing Camp,Camp de pêche +4528,Fish Monitoring Station,Station halieutique +4529,Radar Station,Station radar +4531,Dykes,Digues +4533,Reference Point,Point de référence +4534,Tower,Tour +4535,Mounds,Monticules funéraires +4536,School,École +4539,Control Structure,Barrage +4540,Sanctuary,Sanctuaire +4545,Landmark,Point d'intérêt +4546,Clear,Espace déboisé +4547,Canadian Forces Camp,Camp des Forces canadiennes +4548,Ammunition Depot,Dépôt de munition +4549,Tourist Attraction,Attrait touristique +4550,Indian Camp Site,Camp amerindien +4556,Cemetery,Cimetière +4557,Military Reserve,Réserve militaire +4558,Site,Site +4559,Canadian Forces Base,Base des Forces canadiennes +4560,Canadian Forces Station,Station des Forces canadiennes +4561,Canadian Forces Station,Station des Forces canadiennes +4562,Farm,Ferme +4563,Gate,Porte +4566,Military College,Collège militaire +4567,Military College,Collège militaire +4569,Well,Puit +4571,Holiday Camp,Camp de vacances +4574,Canadian Forces Camp,Camp des Forces canadiennes +4575,Canadian Forces Range and/or Training Area (C.F.T.A.),Champ de tir et/ou champ de manoeuvre (C.M.F.C.) +4576,Canadian Forces Range and/or Training Area (C.F.T.A.),Champ de tir et/ou champ de manoeuvre (C.M.F.C.) +4577,Campground,Terrain de camping +4578,Campground,Terrain de camping +4579,School Camp,Camp-école +4580,Seasonal Camp Sites,Camps saisonniers +4581,Nature Interpretation Centre,Centre d'interprétation de la nature +4582,Hunting and Fishing Club,Club de chasse et de pêche +4583,Summer Camp,Colonie de vacances +4584,Community Pasture,Commune +4585,Estate,Domaine +4586,Ski Resort,Station de ski +4587,Tourist Attraction,Station touristique +4588,Golf Course,Terrain de golf +4589,Playground,Terrain de jeu +4590,Trait-carré,Trait-carré +4592,Administrative Centre,Centre administratif +4593,Dams,Barrages +4594,Registration Building,Barrière +4596,Diversion,Canal de dérivation +4597,Outdoor Camp,Camp de plein air +4598,Outfitting Establishment,Pourvoirie +4599,Floodway,Canal de dérivation +4600,Recreation Centre,Centre récréatif +4601,Museum,Musée +4602,Footbridge,Ponceau +4603,Community College,Collège d'éducation générale et professionnelle (C.E.G.E.P.) +4604,Mound,Monticule funéraire +4605,Dyke,Aboiteau +4606,Commercial Area,Région commerciale +4610,Laboratory,Laboratoire +4613,Hydroelectric Station,Poste de transport +4614,Diversion,Ouvrage de dérivation +4615,Recreation Facility,Établissement récréatif +4617,Fish Trapping Site,Site de piégeage à poissons +4618,Seasonal Camp,Camp saisonnier +4619,Gravesite,Tombe +4620,Correctional Institution,Établissement correctionnel +4621,Canadian Forces Wing,Escadre des Forces canadiennes +4622,Escadre des Forces canadiennes,Canadian Forces Wing +4623,Canadian Forces Base/Area Support Unit,Base des Forces canadiennes/Unité de soutien de secteur +4624,Canadian Forces Base/Area Support Unit,Base des Forces canadiennes/Unité de soutien de secteur +5000,Federal Park,Parc fédéral +5001,Federal Park,Parc fédéral +5002,Region,Région +5003,Region,Région +5004,Crown Lands Reserve,Réserve des terres de la couronne +5005,National Park,Parc national +5006,Protected Beach,Plage protégée +5007,Geopark,Géoparc +5008,Geopark,Géoparc +5009,Heritage site,Site patrimonial +7001,Abyssal Hills,Collines marines +7002,Abyssal Plain,Plaine +7003,Apron,Glacis +7004,Bank,Banc +7005,Banks,Bancs +7006,Bar,Barre +7007,Basin,Bassin +7008,Bench,Terrasse +7009,Borderland,Bordure continentale +7010,Canyon,Canyon +7011,Channel,Chenal +7012,Continental Margin,Précontinent +7013,Continental Margin,Marge continentale +7014,Continental Rise,Glacis continental +7015,Continental Rise,Glacis précontinental +7016,Cordillera,Cordillère +7017,Dome,Dôme +7018,Delta,Delta +7019,Escarpment,Escarpement +7020,Escarpment,Talus +7021,Fan,Cône +7022,Pingos,Pingos +7023,Fracture Zone,Zone de fractures +7024,Furrow,Sillon +7025,Gap,Passage +7027,Guyot,Guyot +7028,Hill,Colline +7029,Hole,Cuvette +7030,Knoll,Dôme +7031,Knolls,Dômes +7032,Ledge,Chaussée +7033,Ledges,Chaussées +7034,Levee,Levée +7035,Lobe,Lobe +7036,Marginal Trough,Cuvette marginale +7037,Median Valley,Vallée axiale +7038,Moat,Douve +7039,Moat,Fossé +7040,Moraine,Moraine +7041,Mountain,Massif +7042,Mountain,Montagne +7043,Pass,Col +7044,Peak,Pic +7045,Pingo,Pingo +7046,Pinnacle,Aiguille +7047,Plateau,Plateau +7048,Province,Province +7049,Province,Région +7050,Reef,Récif +7051,Reefs,Récifs +7052,Ridge,Dorsale +7053,Rise,Massif +7054,Rock,Roche +7055,Rocks,Roches +7056,Saddle,Col +7057,Scarp,Escarpement +7058,Scarp,Talus +7059,Seachannel,Chenal +7060,Seamount Chain,Chaîne de monts +7061,Seamount Chain,Chaînon de monts +7063,Seamount,Mont +7064,Seamounts,Monts sous-marins +7065,Shelf,Plate-forme continentale +7066,Shelf-edge,Rebord de la dorsale +7067,Shelf-edge,Rebord +7068,Shelf,Plate-forme +7069,Shoal,Basse +7070,Shoal,Haut-fond +7071,Shoals,Hauts-fonds +7072,Sill,Seuil +7073,Slope,Pente continentale +7074,Slope,Pente +7075,Spit,Flèche littorale +7076,Spur,Éperon +7077,Tablemount,Guyot +7078,Terrace,Terrasse +7079,Trench,Fossé +7080,Trough,Cuvette +7081,Troughs,Cuvettes +7082,Valley,Vallée +7083,Bars,Barres +7084,Ramp,Talus +8888,Generic Pending Classification,Classification à venir +9999,Unclassified,Non classé diff --git a/docs/lambda/geolocator/geolocator-bucket-content/tables/province.csv b/docs/lambda/geolocator/geolocator-bucket-content/tables/province.csv new file mode 100644 index 00000000..8d72738b --- /dev/null +++ b/docs/lambda/geolocator/geolocator-bucket-content/tables/province.csv @@ -0,0 +1,16 @@ +Code,en,fr +10,Newfoundland and Labrador,Terre-Neuve-et-Labrador +11,Prince Edward Island,Île-du-Prince-Édouard +12,Nova Scotia,Nouvelle-Écosse +13,New Brunswick,Nouveau-Brunswick +24,Quebec,Québec +35,Ontario,Ontario +46,Manitoba,Manitoba +47,Saskatchewan,Saskatchewan +48,Alberta,Alberta +59,British Columbia,Colombie-Britannique +60,Yukon,Yukon +61,Northwest Territories,Territoires du Nord-Ouest +62,Nunavut,Nunavut +72,Undersea Feature,Entité sous-marine +73,International Waters,Eaux internationales diff --git a/docs/lambda/geolocator/geolocator-commit-15b0de5722c2362d49cdfdbdfd2cb9f97c94059e.zip b/docs/lambda/geolocator/geolocator-commit-15b0de5722c2362d49cdfdbdfd2cb9f97c94059e.zip new file mode 100644 index 00000000..6d299333 Binary files /dev/null and b/docs/lambda/geolocator/geolocator-commit-15b0de5722c2362d49cdfdbdfd2cb9f97c94059e.zip differ diff --git a/docs/lambda/id/id-20230529-1030.zip b/docs/lambda/id/id-20230529-1030.zip new file mode 100644 index 00000000..a92912d3 Binary files /dev/null and b/docs/lambda/id/id-20230529-1030.zip differ diff --git a/docs/lambda/id/id-20230823-1330.zip b/docs/lambda/id/id-20230823-1330.zip new file mode 100644 index 00000000..1fd05522 Binary files /dev/null and b/docs/lambda/id/id-20230823-1330.zip differ diff --git a/docs/lambda/id/id-20240612-1600.zip b/docs/lambda/id/id-20240612-1600.zip new file mode 100644 index 00000000..1ec261f5 Binary files /dev/null and b/docs/lambda/id/id-20240612-1600.zip differ diff --git a/docs/lambda/id/id.zip b/docs/lambda/id/id.zip deleted file mode 100644 index ce1eefad..00000000 Binary files a/docs/lambda/id/id.zip and /dev/null differ diff --git a/docs/lambda/id/idv1-20230926-1100.zip b/docs/lambda/id/idv1-20230926-1100.zip new file mode 100644 index 00000000..7f7ce422 Binary files /dev/null and b/docs/lambda/id/idv1-20230926-1100.zip differ diff --git a/docs/lambda/image-processor/image-processor-20240619-1400.zip b/docs/lambda/image-processor/image-processor-20240619-1400.zip new file mode 100644 index 00000000..14186184 Binary files /dev/null and b/docs/lambda/image-processor/image-processor-20240619-1400.zip differ diff --git a/docs/lambda/search-repo/README.md b/docs/lambda/search-repo/README.md new file mode 100644 index 00000000..e5e6f28a --- /dev/null +++ b/docs/lambda/search-repo/README.md @@ -0,0 +1,8 @@ +# OpenSearch # + +The OpenSearch Search Repo is made of several different parts. The opensearch-s3-loader takes metadata from the metadata bucket and loads it into OpenSearch. It then sends PDF, DOCX, and JPEG URLs to the resource_parser Lambda / Gateway which then reaches out to the URL and grabs the resource and sends it through the process of indexing. OpenSearch needs to be created first manually to properly build it. Current Dev instance should be of the right size for GeoDiscovery Needs. + +The presentation for OpenSearch can be found here: https://www.canva.com/design/DAFdR5LVTF8/xdYGDT3jYD7zHWevnWpPkA/view?utm_content=DAFdR5LVTF8&utm_campaign=designshare&utm_medium=link&utm_source=publishsharelink + +Chris Melnick-MacDonald +chris@indiciumx.com diff --git a/docs/lambda/search-repo/addToOSindex.zip b/docs/lambda/search-repo/addToOSindex.zip new file mode 100644 index 00000000..01da88ef Binary files /dev/null and b/docs/lambda/search-repo/addToOSindex.zip differ diff --git a/docs/lambda/search-repo/addToQueueFunction.zip b/docs/lambda/search-repo/addToQueueFunction.zip new file mode 100644 index 00000000..0571d4e6 Binary files /dev/null and b/docs/lambda/search-repo/addToQueueFunction.zip differ diff --git a/docs/lambda/search-repo/addtoOSfroms3.js b/docs/lambda/search-repo/addtoOSfroms3.js new file mode 100644 index 00000000..37b5916e --- /dev/null +++ b/docs/lambda/search-repo/addtoOSfroms3.js @@ -0,0 +1,71 @@ +const AWS = require('aws-sdk'); +const request = require('request'); +const { AWS4Auth } = require('aws4auth'); + +const region = ''; // e.g. us-west-1 +const service = 'es'; +const credentials = new AWS.SharedIniFileCredentials({ profile: 'default' }); +const awsauth = new AWS4Auth({ + accessKeyId: credentials.accessKeyId, + secretAccessKey: credentials.secretAccessKey, + sessionToken: credentials.sessionToken, + region: region, + service: service +}); + +const host = ''; // the OpenSearch Service domain, e.g. https://search-mydomain.us-west-1.es.amazonaws.com +const index = 'lambda-s3-index'; +const datatype = '_doc'; +const url = `${host}/${index}/${datatype}`; + +const headers = { 'Content-Type': 'application/json' }; + +const s3 = new AWS.S3(); + +// Regular expressions used to parse some simple log lines +const ipPattern = /(\d+\.\d+\.\d+\.\d+)/; +const timePattern = /\[(\d+\/\w\w\w\/\d\d\d\d:\d\d:\d\d:\d\d\s-\d\d\d\d)\]/; +const messagePattern = /\"(.+)\"/; + +// Lambda execution starts here +exports.handler = async (event, context) => { + for (const record of event.Records) { + // Get the bucket name and key for the new file + const bucket = record.s3.bucket.name; + const key = record.s3.object.key; + + // Get, read, and split the file into lines + const obj = await s3.getObject({ Bucket: bucket, Key: key }).promise(); + const body = obj.Body.toString(); + const lines = body.split('\n'); + + // Match the regular expressions to each line and index the JSON + for (const line of lines) { + const ipMatch = line.match(ipPattern); + const timestampMatch = line.match(timePattern); + const messageMatch = line.match(messagePattern); + + if (ipMatch && timestampMatch && messageMatch) { + const ip = ipMatch[1]; + const timestamp = timestampMatch[1]; + const message = messageMatch[1]; + + const document = { ip: ip, timestamp: timestamp, message: message }; + const requestOptions = { + url: url, + auth: awsauth, + json: document, + headers: headers + }; + + request.post(requestOptions, (error, response, body) => { + if (error) { + console.error('Error:', error); + } else { + console.log('Response:', body); + } + }); + } + } + } +}; \ No newline at end of file diff --git a/docs/lambda/search-repo/batchingFunction.zip b/docs/lambda/search-repo/batchingFunction.zip new file mode 100644 index 00000000..042c1f96 Binary files /dev/null and b/docs/lambda/search-repo/batchingFunction.zip differ diff --git a/docs/lambda/search-repo/opensearch-s3-event-handler.zip b/docs/lambda/search-repo/opensearch-s3-event-handler.zip new file mode 100644 index 00000000..15cb0ecb Binary files /dev/null and b/docs/lambda/search-repo/opensearch-s3-event-handler.zip differ diff --git a/docs/lambda/search-repo/opensearch-s3-loader.zip b/docs/lambda/search-repo/opensearch-s3-loader.zip new file mode 100644 index 00000000..5e9aedcd Binary files /dev/null and b/docs/lambda/search-repo/opensearch-s3-loader.zip differ diff --git a/docs/lambda/search-repo/processDOCX.zip b/docs/lambda/search-repo/processDOCX.zip new file mode 100644 index 00000000..7d3e853e Binary files /dev/null and b/docs/lambda/search-repo/processDOCX.zip differ diff --git a/docs/lambda/search-repo/processJPG.zip b/docs/lambda/search-repo/processJPG.zip new file mode 100644 index 00000000..357b795e Binary files /dev/null and b/docs/lambda/search-repo/processJPG.zip differ diff --git a/docs/lambda/search-repo/processPDF.zip b/docs/lambda/search-repo/processPDF.zip new file mode 100644 index 00000000..bd5be3a6 Binary files /dev/null and b/docs/lambda/search-repo/processPDF.zip differ diff --git a/docs/lambda/search-repo/queryOSindex.zip b/docs/lambda/search-repo/queryOSindex.zip new file mode 100644 index 00000000..0aef30d5 Binary files /dev/null and b/docs/lambda/search-repo/queryOSindex.zip differ diff --git a/docs/lambda/search-repo/resource_parser.zip b/docs/lambda/search-repo/resource_parser.zip new file mode 100644 index 00000000..0bc801da Binary files /dev/null and b/docs/lambda/search-repo/resource_parser.zip differ diff --git a/docs/lambda/search-repo/samples/000183ed-8864-42f0-ae43-c4313a860720.json b/docs/lambda/search-repo/samples/000183ed-8864-42f0-ae43-c4313a860720.json new file mode 100644 index 00000000..f71599df --- /dev/null +++ b/docs/lambda/search-repo/samples/000183ed-8864-42f0-ae43-c4313a860720.json @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-141.003,41.6755],[-52.6174,41.6755],[-52.6174,83.1139],[-141.003,83.1139],[-141.003,41.6755]]]},"properties":{"id":"000183ed-8864-42f0-ae43-c4313a860720","title":{"en":"Principal Mineral Areas, Producing Mines, and Oil and Gas Fields (900A)","fr":"Principales régions minières, principales mines productrices, principaux champs de pétrole et de gaz (900A)"},"description":{"en":"This dataset is produced and published annually by Natural Resources Canada. It contains a variety of statistics on Canada’s mineral production, and provides the geographic locations of significant metallic, nonmetallic and coal mines, oil sands mines, selected metallurgical works and gas fields for the provinces and territories of Canada.Related product:- **[Top 100 Exploration Projects](https://open.canada.ca/data/en/dataset/b64179f3-ea0f-4abb-9cc5-85432fc958a0)**","fr":"Ce jeu de données est produit et publié annuellement par Ressources naturelles Canada. Il contient une variété de statistiques sur la production de minéraux du Canada et montre l’emplacement géographique des principaux champs de gaz, certaines activités métallurgiques, mines de minéraux métalliques, non métalliques et de charbon, et mines de sables bitumineux dans les provinces et les territoires du Canada.Produit connexe:- **[Les 100 principaux projets d'exploration](https://open.canada.ca/data/fr/dataset/b64179f3-ea0f-4abb-9cc5-85432fc958a0)**"},"keywords":{"en":"mineralization, mineral occurrences, mines, hydrocarbons, fossil fuels, industrial minerals, metallic minerals, economic geology, mineral deposits, exploration and deposit appraisal, nonmetallic minerals, oil, gas, hydrocarbons, refineries, smelters, mineral processing, ferroalloy, automobile shredders, Steel, Gas industry, Coal, Earth sciences, Oil sands, Recycling, Metals, Minerals, Mining industry","fr":"minéralisation, indices minéralisés, mines, hydrocarbures, combustibles fossiles, minéraux industriels, minéraux métalliques, géologie économique, gisements minéraux, exploration et mise en valeur de gisements, minéraux non métalliques, pétrole, gaz, hydrocarbures, raffineries, fonderies, traitement des minerais, ferro-alliages, déchiqueteurs d'automobiles, Acier, Industrie gazière, Charbon, Sciences de la terre, Sables bitumineux, Recyclage, Métal, Minerai, Industrie minière"},"topicCategory":"economy","date":{"published":{"text":"publication; publication","date":"2020-02-27"},"created":{"text":"revision; révision","date":"2021-02"}},"spatialRepresentation":"vector; vecteur","type":"series; série","geometry":"POLYGON((-141.003 41.6755, -52.6174 41.6755, -52.6174 83.1139, -141.003 83.1139, -141.003 41.6755))","temporalExtent":{"begin":"2020-01","end":"2020-12"},"refSys":"EPSG:3978","refSys_version":"8.9.2","status":"completed; complété","maintenance":"annually; annuel","metadataStandard":{"en":"North American Profile of ISO 19115:2003 - Geographic information - Metadata","fr":"Profil nord-américain de la norme ISO 19115:2003 - Information géographique - Métadonnées"},"metadataStandardVersion":"CAN/CGSB-171.100-2009","graphicOverview":[{"overviewFileName":"http://ftp.maps.canada.ca/pub/nrcan_rncan/Mining-industry_Industrie-miniere/900A_and_top_100/Thumbnail/900A.png","overviewFileDescription":null,"overviewFileTupe":null}],"distributionFormat_name":null,"distributionFormat_format":null,"useLimits":{"en":"Open Government Licence - Canada (http://open.canada.ca/en/open-government-licence-canada)","fr":"Licence du gouvernement ouvert - Canada (http://ouvert.canada.ca/fr/licence-du-gouvernement-ouvert-canada)"},"accessConstraints":"license; licence","otherConstraints":{"en":null,"fr":null},"dateStamp":"2021-09-24T13:44:48","dataSetURI":null,"locale":{"language":"French; Français","country":"Canada; Canada","encoding":"utf8; utf8"},"language":"eng; CAN","characterSet":"utf8; utf8","environmentDescription":null,"supplementalInformation":{"en":null,"fr":null},"contact":[{"individual":null,"position":{"en":null,"fr":null},"organisation":{"en":"Government of Canada; Natural Resources Canada; Lands and Minerals Sector","fr":"Gouvernement du Canada; Ressources naturelles Canada; Secteur des terres et des minéraux"},"telephone":{"en":"1-855-525-9293","fr":"1-855-525-9293"},"fax":null,"address":{"en":"580 Booth Street","fr":"580 rue Booth"},"city":"Ottawa","pt":{"en":"Ontario","fr":"Ontario"},"postalcode":"K1A 0E4","country":{"en":"Canada","fr":"Canada"},"email":{"en":"NRCan.questions.RNCan@canada.ca","fr":"NRCan.questions.RNCan@canada.ca"},"onlineResource":{"onlineResource":null,"onlineResource_Name":null,"onlineResource_Protocol":"http","onlineResource_Description":null},"hoursOfService":null,"role":"pointOfContact; contact"}],"credits":[],"cited":[{"individual":null,"position":{"en":null,"fr":null},"organisation":{"en":"Government of Canada; Natural Resources Canada; Lands and Minerals Sector","fr":"Gouvernement du Canada; Ressources naturelles Canada; Secteur des terres et des minéraux"},"telephone":{"en":"1-855-525-9293","fr":"1-855-525-9293"},"fax":null,"address":{"en":"580 Booth Street","fr":"580 rue Booth"},"city":"Ottawa","pt":{"en":"Ontario","fr":"Ontario"},"postalcode":"K1A 0E4","country":{"en":"Canada","fr":"Canada"},"email":{"en":"NRCan.questions.RNCan@canada.ca","fr":"NRCan.questions.RNCan@canada.ca"},"onlineResource":{"onlineResource":null,"onlineResource_Name":null,"onlineResource_Protocol":"http","onlineResource_Description":null},"hoursOfService":null,"role":"pointOfContact; contact"}],"distributor":[{"individual":null,"position":{"en":null,"fr":null},"organisation":{"en":"Government of Canada; Natural Resources Canada; Lands and Minerals Sector","fr":"Gouvernement du Canada; Ressources naturelles Canada; Secteur des terres et des minéraux"},"telephone":{"en":"1-855-525-9293","fr":"1-855-525-9293"},"fax":null,"address":{"en":"580 Booth Street","fr":"580 rue Booth"},"city":"Ottawa","pt":{"en":null,"fr":"Ontario"},"postalcode":"K1A 0E4","country":{"en":"Canada","fr":"Canada"},"email":{"en":"NRCan.questions.RNCan@canada.ca","fr":"NRCan.questions.RNCan@canada.ca"},"onlineResource":{"onlineResource":null,"onlineResource_Name":null,"onlineResource_Protocol":"http","onlineResource_Description":null},"hoursOfService":null,"role":"distributor; distributeur"}],"options":[{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":"https://ftp.maps.canada.ca/pub/nrcan_rncan/Mining-industry_Industrie-miniere/900A_and_top_100/900A.gdb.zip","protocol":"HTTPS","name":{"en":"Vector dataset of the 900A map","fr":"Jeu de données vectorielles de la carte 900A"},"description":{"en":"Dataset;FGDB/GDB;eng,fra","fr":"Données;FGDB/GDB;eng,fra"}},{"url":"https://ftp.maps.canada.ca/pub/nrcan_rncan/Mining-industry_Industrie-miniere/900A_and_top_100/900A_and_top_100_en.mxd","protocol":"HTTPS","name":{"en":"ArcMap project file of the 900A web map","fr":"Projet ArcMap de la cartographie web de 900A"},"description":{"en":"Supporting Document;MXD;eng","fr":"Document de soutien;MXD;eng"}},{"url":"https://ftp.maps.canada.ca/pub/nrcan_rncan/Mining-industry_Industrie-miniere/900A_and_top_100/900A_and_top_100_fr.mxd","protocol":"HTTPS","name":{"en":"ArcMap project file of the 900A web map","fr":"Projet ArcMap de la cartographie web de 900A"},"description":{"en":"Supporting Document;MXD;fra","fr":"Document de soutien;MXD;fra"}},{"url":"https://ftp.maps.canada.ca/pub/nrcan_rncan/Mining-industry_Industrie-miniere/900A_and_top_100/900A_70th_shape.zip","protocol":"HTTPS","name":{"en":"Vector dataset of the 900A map","fr":"Jeu de données vectorielles de la carte 900A"},"description":{"en":"Dataset;SHP;eng,fra","fr":"Données;SHP;eng,fra"}},{"url":"https://ftp.maps.canada.ca/pub/nrcan_rncan/Mining-industry_Industrie-miniere/900A_and_top_100/900a_70th_English.pdf","protocol":"HTTPS","name":{"en":"Mineral Areas, Producing Mines, and Oil and Gas Fields (900A)","fr":"Principales régions minières, principales mines productrices, principaux champs de pétrole et de gaz (900A)"},"description":{"en":"Supporting Document;PDF;eng","fr":"Document de soutien;PDF;eng"}},{"url":"https://ftp.maps.canada.ca/pub/nrcan_rncan/Mining-industry_Industrie-miniere/900A_and_top_100/900a_70th_French.pdf","protocol":"HTTPS","name":{"en":"Mineral Areas, Producing Mines, and Oil and Gas Fields (900A)","fr":"Principales régions minières, principales mines productrices, principaux champs de pétrole et de gaz (900A)"},"description":{"en":"Supporting Document;PDF;fra","fr":"Document de soutien;PDF;fra"}},{"url":"https://maps-cartes.services.geo.ca/server_serveur/rest/services/NRCan/900A_and_top_100_en/MapServer/1","protocol":"ESRI REST: Map Server","name":{"en":"Mineral Areas, Producing Mines, and Oil and Gas Fields (900A)","fr":"Principales régions minières, principales mines productrices, principaux champs de pétrole et de gaz (900A)"},"description":{"en":"Web Service;ESRI REST;eng","fr":"Service Web;ESRI REST;eng"}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}}]}}]} \ No newline at end of file diff --git a/docs/lambda/search-repo/samples/0005301b-624e-4000-8dad-a1a1ac6b46c2.json b/docs/lambda/search-repo/samples/0005301b-624e-4000-8dad-a1a1ac6b46c2.json new file mode 100644 index 00000000..5c51d9ce --- /dev/null +++ b/docs/lambda/search-repo/samples/0005301b-624e-4000-8dad-a1a1ac6b46c2.json @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-141.003,41.6755],[-52.6174,41.6755],[-52.6174,83.1139],[-141.003,83.1139],[-141.003,41.6755]]]},"properties":{"id":"0005301b-624e-4000-8dad-a1a1ac6b46c2","title":{"en":"Land Surface Evapotranspiration for Canada's Landmass","fr":"Évapotranspiration de la surface terrestre continentale du Canada"},"description":{"en":"The dataset contains land surface evapotranspiration (ET) for Canada's landmass at a spatial resolution of 5-km. The values represent the annual total amount of ET in mm of H2O, averaged over a 38-year period of 1979-2016. The ET was produced by the Land Surfae Model EALCO (Ecological Assimilation of Land and CLimate Observations) developed at Natural Resources Canada. Details of the dataset can be found in Wang et al. (2013, Spatial and seasonal variations in evapotranspiration over Canada’s landmass. Hydrol. Earth Syst. Sci., 17, 3561–3575, doi:10.5194/hess-17-3561-2013) and Wang (2007, Simulation of Evapotranspiration and Its Response to Plant Water and CO2 Transfer Dynamics. J. Hydrometeorology, 9, 426-443, DOI: 10.1175/2007JHM918.1).","fr":"Le jeu de données représente l'évapotranspiration (ET) de la surface terrestre continentale du Canada à une résolution spatiale de 5 km. Les valeurs représentent la quantité totale annuelle d'ET en mm d'H2O, moyenne sur une période de 38 ans de 1979 à 2016. L'ET a été produit par Modélisation des écosystèmes et assimilation des données satellite (modèle AEOCT : Assimilation écologique des observations terrestres et climatiques) développé par Ressources naturelles Canada. Les détails de l'ensemble de données peuvent être trouvés dans Wang et al., 2013 (Variations spatiales et saisonnières de l'évapotranspiration dans la masse continentale du Canada. Hydrol. Earth Syst. Sci., 17, 3561–3575, doi: 10.5194 / hess-17-3561-2013) et Wang, 2007 (Simulation of Evapotranspiration and Its Response to Plant Water and CO2 Transfer Dynamics. J. Hydrometeorology, 9, 426-443, DOI: 10.1175 / 2007JHM918.1)."},"keywords":{"en":"Land, Atmosphere, Ecosystems, Water, Meteorology, Databases, Climate, Climate, ","fr":"Terre, Atmosphère, Écosystème, Eau, Météorologie, Base de données, Climat, Climat, "},"topicCategory":"geoscientificInformation","date":{"published":{"text":"publication; publication","date":"2013-09-23"},"created":{"text":"revision; révision","date":"2019-02-01"}},"spatialRepresentation":"grid; grille","type":"dataset; jeuDonnées","geometry":"POLYGON((-141.003 41.6755, -52.6174 41.6755, -52.6174 83.1139, -141.003 83.1139, -141.003 41.6755))","temporalExtent":{"begin":"1979","end":"2016"},"refSys":"EPSG:3978","refSys_version":null,"status":"completed; complété","maintenance":"irregular; irrégulier","metadataStandard":{"en":"North American Profile of ISO 19115:2003 - Geographic information - Metadata","fr":"Profil nord-américain de la norme ISO 19115:2003 - Information géographique - Métadonnées"},"metadataStandardVersion":"CAN/CGSB-171.100-2009","graphicOverview":[{"overviewFileName":"https://csw.open.canada.ca/geonetwork/srv/api/records/0005301b-624e-4000-8dad-a1a1ac6b46c2/attachments/2020_03_19_10_10_55_Clipboard_s.png","overviewFileDescription":"thumbnail","overviewFileType":"png; png"},{"overviewFileName":"https://csw.open.canada.ca/geonetwork/srv/api/records/0005301b-624e-4000-8dad-a1a1ac6b46c2/attachments/2020_03_19_10_10_55_Clipboard.png","overviewFileDescription":"large_thumbnail","overviewFileType":"png; png"},{"overviewFileName":"https://csw.open.canada.ca/geonetwork/srv/api/records/0005301b-624e-4000-8dad-a1a1ac6b46c2/attachments/2020_03_19_10_10_55_Clipboard_s_fre.png","overviewFileDescription":"thumbnail_fre","overviewFileType":"png; png"},{"overviewFileName":"https://csw.open.canada.ca/geonetwork/srv/api/records/0005301b-624e-4000-8dad-a1a1ac6b46c2/attachments/2020_03_19_10_10_55_Clipboard.png","overviewFileDescription":"large_thumbnail_fre","overviewFileType":"png; png"}],"distributionFormat_name":null,"distributionFormat_format":null,"useLimits":{"en":"Open Government Licence - Canada (http://open.canada.ca/en/open-government-licence-canada)","fr":"Licence du gouvernement ouvert - Canada (http://ouvert.canada.ca/fr/licence-du-gouvernement-ouvert-canada)"},"accessConstraints":"license; licence","otherConstraints":{"en":null,"fr":null},"dateStamp":"2021-09-08T13:51:12","dataSetURI":null,"locale":{"language":"French; Français","country":"Canada; Canada","encoding":"utf8; utf8"},"language":"eng; CAN","characterSet":"utf8; utf8","environmentDescription":null,"supplementalInformation":{"en":null,"fr":null},"contact":[{"individual":"Wang, Shusen","position":{"en":"Research Scientist","fr":"Chercheur"},"organisation":{"en":"Government of Canada; Natural Resources Canada; Canada Centre for Remote Science (CCRS)","fr":"Gouvernement du Canada; Ressources naturelles Canada; Centre canadien de télédétection (CCT)"},"telephone":{"en":null,"fr":null},"fax":null,"address":{"en":null,"fr":null},"city":null,"pt":{"en":null,"fr":null},"postalcode":null,"country":{"en":null,"fr":null},"email":{"en":"shusen.wang@canada.ca","fr":"shusen.wang@canada.ca"},"onlineResource":{"onlineResource":null,"onlineResource_Name":null,"onlineResource_Protocol":"http","onlineResource_Description":null},"hoursOfService":null,"role":"pointOfContact; contact"}],"credits":[],"cited":[{"individual":"Wang, Shusen","position":{"en":"Research Scientist","fr":"Chercheur"},"organisation":{"en":"Government of Canada; Natural Resources Canada; Canada Centre for Remote Science (CCRS)","fr":"Gouvernement du Canada; Ressources naturelles Canada; Centre canadien de télédétection (CCT)"},"telephone":{"en":null,"fr":null},"fax":null,"address":{"en":null,"fr":null},"city":null,"pt":{"en":null,"fr":null},"postalcode":null,"country":{"en":null,"fr":null},"email":{"en":"shusen.wang@canada.ca","fr":"shusen.wang@canada.ca"},"onlineResource":{"onlineResource":null,"onlineResource_Name":null,"onlineResource_Protocol":"http","onlineResource_Description":null},"hoursOfService":null,"role":"author; auteur"}],"distributor":[{"individual":null,"position":{"en":null,"fr":null},"organisation":{"en":"Government of Canada; Natural Resources Canada; Canada Centre for Mapping and Earth Observation (CCMEO)","fr":"Gouvernement du Canada; Ressources naturelles Canada; Centre canadien de cartographie et d'observation de la Terre (CCCOT)"},"telephone":{"en":"1-800-661-2638","fr":"1-800-661-2638"},"fax":null,"address":{"en":null,"fr":null},"city":null,"pt":{"en":null,"fr":null},"postalcode":null,"country":{"en":null,"fr":null},"email":{"en":"NRCan.fgp-pgf.RNCan@Canada.ca","fr":"NRCan.fgp-pgf.RNCan@Canada.ca"},"onlineResource":{"onlineResource":null,"onlineResource_Name":null,"onlineResource_Protocol":"http","onlineResource_Description":null},"hoursOfService":null,"role":"distributor; distributeur"}],"options":[{"url":"https://maps-cartes.services.geo.ca/server_serveur/rest/services/NRCan/land_surface_evapotranspiration_en/MapServer","protocol":"ESRI REST: Map Server","name":{"en":"Land Surface Evapotranspiration for Canada Landmass","fr":"Évapotranspiration de la surface terrestre continentale du Canada"},"description":{"en":"Web Service;ESRI REST;eng","fr":"Service Web;ESRI REST;eng"}},{"url":"https://maps-cartes.services.geo.ca/server_serveur/rest/services/NRCan/land_surface_evapotranspiration_fr/MapServer","protocol":"ESRI REST: Map Server","name":{"en":"Land Surface Evapotranspiration for Canada Landmass","fr":"Évapotranspiration de la surface terrestre continentale du Canada"},"description":{"en":"Web Service;ESRI REST;fra","fr":"Service Web;ESRI REST;fra"}},{"url":"https://maps-cartes.services.geo.ca/server_serveur/services/NRCan/land_surface_evapotranspiration_en/MapServer/WMSServer?request=GetCapabilities&service=WMS&version=1.3.0&layers=0&legend_format=image%2Fpng&feature_info_type=text%2Fhtml","protocol":"OGC:WMS","name":{"en":"Land Surface Evapotranspiration for Canada Landmass","fr":"Évapotranspiration de la surface terrestre continentale du Canada"},"description":{"en":"Web Service;WMS;eng","fr":"Service Web;WMS;eng"}},{"url":"https://maps-cartes.services.geo.ca/server_serveur/services/NRCan/land_surface_evapotranspiration_fr/MapServer/WMSServer?request=GetCapabilities&service=WMS&version=1.3.0&layers=0&legend_format=image%2Fpng&feature_info_type=text%2Fhtml","protocol":"OGC:WMS","name":{"en":"Land Surface Evapotranspiration for Canada Landmass","fr":"Évapotranspiration de la surface terrestre continentale du Canada"},"description":{"en":"Web Service;WMS;fra","fr":"Service Web;WMS;fra"}},{"url":"https://ftp.maps.canada.ca/pub/nrcan_rncan/Meteorology_Meteorologie/land_surface_evapotranspiration-evapotranspiration_surface_terrestre/YET_Mean_Land_Surface_Evapotranspiration.tif","protocol":"HTTP","name":{"en":"Land Surface Evapotranspiration for Canada Landmass","fr":"Évapotranspiration de la surface terrestre continentale du Canada"},"description":{"en":"Dataset;TIFF;zxx","fr":"Données;TIFF;zxx"}}]}}]} \ No newline at end of file diff --git a/docs/lambda/search-repo/samples/001c30bd-0869-22a7-ef10-2284e8f9b560.json b/docs/lambda/search-repo/samples/001c30bd-0869-22a7-ef10-2284e8f9b560.json new file mode 100644 index 00000000..768b957a --- /dev/null +++ b/docs/lambda/search-repo/samples/001c30bd-0869-22a7-ef10-2284e8f9b560.json @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-144.76137331,31.61920424],[-9.31597146,31.61920424],[-9.31597146,83.47936456],[-144.76137331,83.47936456],[-144.76137331,31.61920424]]]},"properties":{"id":"f612e2b4-5c67-46dc-9a84-1154c649ab4e","title":{"en":"Atlas of Seabirds at Sea in Eastern Canada 2006 - 2020","fr":"Atlas des oiseaux en mer dans l'est du Canada 2006 - 2020"},"description":{"en":"The atlas provides printable maps, Web Services and downloadable data files representing seabirds at-sea densities in eastern Canada.\\n\\nThe information provided on the open data web site can be used to identify areas where seabirds at sea are found in eastern Canada. However, low survey effort or high variation in some areas introduces uncertainty in the density estimates provided. The data and maps found on the open data web site should therefore be interpreted with an understanding of this uncertainty.\\n\\nData were collected using ships of opportunity surveys and therefore spatial and seasonal coverage varies considerably. Densities are computed using distance sampling to adjust for variation in detection rates among observers and survey conditions. Depending on conditions, seabirds can be difficult to identify to species level. Therefore, densities at higher taxonomic levels are provided. more details in the document: Atlas_SeabirdsAtSea-OiseauxMarinsEnMer.pdf.\\n\\nBy clicking on \"View on Map\" you will visualize a example of the density measured for all species combined from April to July - 2006-2020. ESRI REST or WMS map services can be added to your web maps or opened directly in your desktop mapping applications. These are alternatives to downloading and provide densities for all taxonomical groups and species as well as survey effort.","fr":"L'atlas donne accès à des cartes imprimables, des services de données Web et des fichiers de données téléchargeables sur les densités d'oiseaux en mer de l'est du Canada.\\n\\nL’information fournie sur le site des données ouvertes peut être utilisée pour identifier les endroits où les oiseaux en mer se retrouvent dans l’est du Canada. Cependant, le faible effort d’échantillonnage et la grande variabilité à certains endroits y entraînent une plus grande incertitude des résultats. Ainsi, les données et les cartes accessibles sur le site des données ouvertes devraient être consultées en toute connaissance de cause.\\n\\nLes données ont été recueillies sur des navires occasionnels et donc la couverture spatiale et saisonnière varie considérablement. Les densités ont été calculées en utilisant l'échantillonnage par distance afin de réaliser des ajustements liés à la détectabilité variable entre les observateurs et les conditions d'observation. Puisque souvent les oiseaux peuvent être difficile à identifier à l'espèce dépendamment des conditions observation, les densités d'oiseaux en mer sont présentées à divers niveaux taxonomiques. Vous trouverez plus de détails dans le document Atlas_SeabirdsAtSea-OiseauxMarinsEnMer.pdf\\n\\nEn cliquant sur \"Afficher la carte\", vous visualiserez un exemple pour la densité mesurée pour toutes les espèces combinées pour la période d'avril à juillet - 2006-2020. Les services de carte ESRI REST ou WMS peuvent être ajoutés à vos cartes Web ou ouverts directement dans vos applications cartographiques de bureau. Ce sont des alternatives au téléchargement et fournissent les densités pour tous les groupes taxonomiques, et espèces de même qu'un aperçu de l'effort d'inventaire."},"keywords":{"en":"inventory effort; bird density; Alcid; Gull;Jaegers; murres; murre; phalarope; shearwater; puffin; storm-petrel; oceanite; tern; skuas; labbe; atlantic puffin; black guillemot; cory's shearwater; common tern; dovekie; great black-backed gull; great shearwater; shearwater; herring gull; iceland gull; leach's storm-petrel; océanite cul-blanc; long-tailed jaeger; jaeger; labbe; labbe à longue queue; manx shearwater; puffin des anglais; northern fulmar; fulmar boréal; fulmar; northern gannet; fou de bassan; pomarine jaeger; labbe pomarin; raxorbill; petit pingouin; ring-billed gull; gull; red phalarope; red-throated loon; plongeon catmarin; sooty shearwater; thick-billed murre; wilson's storm petrel; océanite de wilson; black-legged kittiwake; mouette tridactyle; bonaparte's gull; common loon ; devekie; Seabirds; Migratory birds; Ocean; Oceans, Scientific information, Animals, Aquatic birds, Animal migrations, Nature and Biodiversity, National (CA), Assess Status of Species, Develop Species Recovery / Management / Conservation Plans, Protect Species Well-Being, Expand Scientific Knowledge / Develop New Methodologies for Assessing Site Conditions, Species, Canadian Wildlife Service, 1.1.3. Migratory Birds, Unclassified","fr":"effort d'inventaire; densité d'oiseau; espèce; espece; Alcidé; Goéland; Petit labbe; murres; murre; guillemot marmette; guillemot de Brunnich; guillemot de Brünnich; phalarope; puffin;oceanite; océanite;sterne;skuas;grand labbe; labbe;macareux moine;guillemot marmette; puffin cendré; puffin cendre; sterne pierregarin; mergule nain; goéland bourgmestre; puffin majeur; goeland; goéland; goéland argenté; goeland argente; goéland arctique; goeland arctique; goéland à bec cerclé; phalarope à bec large; puffin fuligineux; guillemot de brünnich; mouette de bonaparte; plongeon huard; mergule nain; Oiseaux marins, oiseau marin; oiseaux migrateurs; oiseau migrateur; océan; ocean, Information scientifique, Animal, Oiseau aquatique, Migration animale, Nature et biodiversité, National (CA), Évaluer l'état des espèces, Élaborer des plans de rétablissement, de gestion et de conservation des espèces, Protéger le bien-être des espèces, Approfondir les connaissances scientifiques et formuler de nouvelles méthodes pour gérer, protéger et rétablir les espèces, Espèce, Service canadien de la faune, 1.1.3 Oiseaux migrateurs, Non Classifié"},"topicCategory":"biota","parentIdentifier":null,"date":{"published":{"text":"publication; publication","date":"2022-05-31"},"created":{"text":"creation; création","date":"2016-11-15"},"revision":{"text":null,"date":null},"notavailable":{"text":null,"date":null},"inforce":{"text":null,"date":null},"adopted":{"text":null,"date":null},"deprecated":{"text":null,"date":null},"superceded":{"text":null,"date":null}},"spatialRepresentation":"vector; vecteur","type":"dataset; jeuDonnées","geometry":"POLYGON((-144.76137331 31.61920424, -9.31597146 31.61920424, -9.31597146 83.47936456, -144.76137331 83.47936456, -144.76137331 31.61920424))","temporalExtent":{"begin":"2006-06-01","end":"2020-06-17"},"refSys":"EPSG:4326","refSys_version":"Esri","status":"completed; complété","maintenance":"asNeeded; auBesoin","metadataStandard":{"en":"North American Profile of ISO 19115:2003 - Geographic information - Metadata","fr":"Profil nord-américain de la norme ISO 19115:2003 - Information géographique - Métadonnées"},"metadataStandardVersion":"Enviroment Canada - Version 1","graphicOverview":[{"overviewFileName":"https://csw.open.canada.ca/geonetwork/srv/api/records/f612e2b4-5c67-46dc-9a84-1154c649ab4e/attachments/OverviewAOEM.PNG","overviewFileDescription":"OverviewAOEM.PNG","overviewFileTupe":null}],"distributionFormat_name":"FGDB/GDB","distributionFormat_format":"Unknown/Inconnue","useLimits":{"en":"Open Government Licence - Canada (http://open.canada.ca/en/open-government-licence-canada)","fr":"Licence du gouvernement ouvert - Canada (http://ouvert.canada.ca/fr/licence-du-gouvernement-ouvert-canada)"},"accessConstraints":"license; licence","otherConstraints":{"en":null,"fr":null},"dateStamp":"2023-02-15T13:08:43","dataSetURI":null,"locale":{"language":"French; Français","country":"Canada; Canada","encoding":"utf8; utf8"},"language":"eng; CAN","characterSet":"utf8; utf8","environmentDescription":null,"supplementalInformation":{"en":null,"fr":null},"contact":[{"individual":"Centre de renseignements à la population / Public inquiries centre","position":{"en":"Public inquiries centre","fr":"Centre de renseignements à la population"},"organisation":{"en":"Government of Canada; Environment and Climate Change Canada; Regional Operations, Quebec Region","fr":"Gouvernement du Canada; Environnement et Changement climatique Canada; Opérations régionales, région du Québec"},"telephone":{"en":"1-800-668-6767","fr":"1-800-668-6767"},"fax":null,"address":{"en":"Fontaine Building 12th floor, 200 Sacré-Coeur Blvd","fr":"200 Sacré-Coeur Blvd, Édifice Fontaine 12e étage"},"city":"Gatineau","pt":{"en":"Quebec","fr":"Québec"},"postalcode":"K1A 0H3","country":{"en":"Canada","fr":"Canada (le)"},"email":{"en":"enviroinfo@ec.gc.ca","fr":"ec.enviroinfo.ec@canada.ca"},"onlineResource":{"onlineResource":"http://ec.gc.ca","onlineResource_Name":null,"onlineResource_Protocol":"http","onlineResource_Description":null},"hoursOfService":null,"role":"pointOfContact; contact"}],"credits":[],"distributor":[{"individual":"Environment and Climate Change Canada","position":{"en":"Public inquiries centre","fr":"Centre de renseignements à la population"},"organisation":{"en":"Government of Canada; Environment and Climate Change Canada; Canadian Wildlife Services","fr":"Gouvernement du Canada; Environnement et Changement climatique Canada; Service canadien de la faune"},"telephone":{"en":"1-800-668-6767","fr":"1-800-668-6767"},"fax":null,"address":{"en":"Fontaine Building 12th floor, 200 Sacré-Coeur Blvd","fr":"200 Sacré-Coeur Blvd, Édifice Fontaine 12e étage"},"city":"Gatineau","pt":{"en":null,"fr":"Québec"},"postalcode":"K1A 0H3","country":{"en":"Canada","fr":"Canada (le)"},"email":{"en":"ec.enviroinfo.ec@canada.ca","fr":"ec.enviroinfo.ec@canada.ca"},"onlineResource":{"onlineResource":"http://ec.gc.ca","onlineResource_Name":null,"onlineResource_Protocol":"http","onlineResource_Description":null},"hoursOfService":null,"role":"distributor; distributeur"}],"options":[{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/Atlas_SeabirdsAtSea-OiseauxMarinsEnMer.pdf","protocol":"HTTPS","name":{"en":"Metadata - Presentation of the Project","fr":"Métadonnées - présentation du projet"},"description":{"en":"Supporting Document;PDF;eng,fra","fr":"Document de soutien;PDF;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/CalenPresen_GlSL.pdf","protocol":"HTTPS","name":{"en":"Calendar : Presence in Gulf of Saint Lawrence bioregion","fr":"Calendrier: Présence dans la biorégion du Golfe du Saint−Laurent"},"description":{"en":"Supporting Document;PDF;eng,fra","fr":"Document de soutien;PDF;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/CalenPresen_NLS.pdf","protocol":"HTTPS","name":{"en":"Calendar: Presence in Newfoundland−Labrador Shelves bioregion","fr":"Calendrier: Présence dans la biorégion des Plates−formes de Terre−Neuve et du Labrador"},"description":{"en":"Supporting Document;PDF;eng,fra","fr":"Document de soutien;PDF;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/CalenPresen_SS.pdf","protocol":"HTTPS","name":{"en":"Calendar: Presence in Scotian Shelfs bioregion","fr":"Calendrier: présence dans la biorégion de la Plate−forme Scotian"},"description":{"en":"Supporting Document;PDF;eng,fra","fr":"Document de soutien;PDF;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/data_AOEM_gdb.zip","protocol":"HTTPS","name":{"en":"Data - file geodatabase","fr":"Données - file geodatabase"},"description":{"en":"Dataset;FGDB/GDB;eng,fra","fr":"Données;FGDB/GDB;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/data_AOEM_shp.zip","protocol":"HTTPS","name":{"en":"Data - shapefile","fr":"Données - shapefile"},"description":{"en":"Dataset;SHP;eng,fra","fr":"Données;SHP;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/DetectionModels-ModelesDeDetection.pdf","protocol":"HTTPS","name":{"en":"Metadata: Detection Model","fr":"Métadonnées: Modèle de détection"},"description":{"en":"Supporting Document;PDF;eng,fra","fr":"Document de soutien;PDF;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/FieldsDensity_ChampsDensite.csv","protocol":"HTTPS","name":{"en":"Metadata - Details Field Density","fr":"Métadonnées - Détails champ densité"},"description":{"en":"Supporting Document;CSV;eng,fra","fr":"Document de soutien;CSV;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/FieldsEffort_ChampsEffort.csv","protocol":"HTTPS","name":{"en":"Metadata - Details Field Effort","fr":"Métadonnées - Détails champ effort"},"description":{"en":"Supporting Document;CSV;eng,fra","fr":"Document de soutien;CSV;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/ListGroups_Groupes.csv","protocol":"HTTPS","name":{"en":"Metadata - Detail List Group","fr":"Metadonnées - Détails liste des groupes"},"description":{"en":"Supporting Document;CSV;eng,fra","fr":"Document de soutien;CSV;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/ListSpecies_Especes.csv","protocol":"HTTPS","name":{"en":"Metadata - Details List Species","fr":"Métadonnées - Détails liste des espèces"},"description":{"en":"Supporting Document;CSV;eng,fra","fr":"Document de soutien;CSV;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/MapsCartes_DensityEspeces.zip","protocol":"HTTPS","name":{"en":"Maps - Density by Species","fr":"Cartes - Densité par espèce"},"description":{"en":"Supporting Document;PNG;eng,fra","fr":"Document de soutien;PNG;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/MapsCartes_DensityGroups.zip","protocol":"HTTPS","name":{"en":"Maps - Density by Groups","fr":"Cartes - Densité par groupe"},"description":{"en":"Supporting Document;PNG;eng,fra","fr":"Document de soutien;PNG;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/MapsCartes_Effort.zip","protocol":"HTTPS","name":{"en":"Maps - Inventory Effort","fr":"Cartes - Effort d'inventaire"},"description":{"en":"Supporting Document;PNG;eng,fra","fr":"Document de soutien;PNG;eng,fra"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/SOMEC-ECSAS_Protocol_En.pdf","protocol":"HTTPS","name":{"en":"Protocol - ECSAS (en)","fr":"Protocole - SOMEC (en)"},"description":{"en":"Supporting Document;PDF;eng","fr":"Document de soutien;PDF;eng"}},{"url":"https://data-donnees.ec.gc.ca/data/species/assess/atlas-of-seabirds-at-sea-in-eastern-canada/SOMEC-ECSAS_Protocole_Fr.pdf","protocol":"HTTPS","name":{"en":"Protocol - ECSAS (fr)","fr":"Protocole - SOMEC (fr)"},"description":{"en":"Supporting Document;PDF;fra","fr":"Document de soutien;PDF;fra"}},{"url":"https://maps-cartes.ec.gc.ca/arcgis/rest/services/CWS_SCF/AtlasOiseauxMerEstCanada/MapServer","protocol":"HTTPS","name":{"en":"Web service (FR) Atlas of Birds at Sea in Eastern Canada 2006 - 2020","fr":"Service Web (FR) Atlas des oiseaux en mer dans l'est du Canada 2006 - 2020"},"description":{"en":"Web Service;ESRI REST;fra","fr":"Service Web;ESRI REST;fra"}},{"url":"https://maps-cartes.ec.gc.ca/arcgis/rest/services/CWS_SCF/AtlasSeabirdsEasternCanada/MapServer","protocol":"HTTPS","name":{"en":"Web service (EN) Atlas of Birds at Sea in Eastern Canada 2006 - 2020","fr":"Service Web (EN) Atlas des oiseaux en mer dans l'est du Canada 2006 - 2020"},"description":{"en":"Web Service;ESRI REST;eng","fr":"Service Web;ESRI REST;eng"}},{"url":"https://maps-cartes.ec.gc.ca/arcgis/services/CWS_SCF/AtlasSeabirdsEasternCanada/MapServer/WMSServer?request=GetCapabilities&service=WMS&layers=All_species_-_Density_by_taxonomic_groups_-_December_to_March65532","protocol":"OGC:WMS","name":{"en":"WMS (en) Atlas of Birds at Sea in Eastern Canada 2006 - 2020","fr":"WMS (en) Atlas des oiseaux en mer dans l'est du Canada 2006 - 2020"},"description":{"en":"Web Service;WMS;eng","fr":"Service Web;WMS;eng"}},{"url":"https://maps-cartes.ec.gc.ca/arcgis/services/CWS_SCF/AtlasOiseauxMerEstCanada/MapServer/WMSServer?request=GetCapabilities&service=WMS&layers=90","protocol":"OGC:WMS","name":{"en":"WMS (fr) Atlas of Birds at Sea in Eastern Canada 2006 - 2020","fr":"WMS (fr) Atlas des oiseaux en mer dans l'est du Canada 2006 - 2020"},"description":{"en":"Web Service;WMS;fra","fr":"Service Web;WMS;fra"}},{"url":"https://maps-cartes.ec.gc.ca/arcgis/rest/services/CWS_SCF/AtlasOiseauxMerEstCanada/MapServer/5","protocol":"ESRI REST: Map Service","name":{"en":"(fr) All species - Density by taxonomic groups - April to July","fr":"(fr) Toutes les espèces - Densité par groupe taxonomique - avril à juillet"},"description":{"en":"Web Service;ESRI REST;fra","fr":"Service Web;ESRI REST;fra"}},{"url":"https://maps-cartes.ec.gc.ca/arcgis/rest/services/CWS_SCF/AtlasSeabirdsEasternCanada/MapServer/5","protocol":"ESRI REST: Map Service","name":{"en":"(en) All species - Density by taxonomic groups - April to July","fr":"(en) Toutes les espèces - Densité par groupe taxonomique - avril à juillet"},"description":{"en":"Web Service;ESRI REST;eng","fr":"Service Web;ESRI REST;eng"}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}},{"url":null,"protocol":null,"name":{"en":null,"fr":null},"description":{"en":null,"fr":null}}]}}]} \ No newline at end of file diff --git a/docs/lambda/search-repo/samples/00215aac-e0af-def2-9907-c08ae38291e4.json b/docs/lambda/search-repo/samples/00215aac-e0af-def2-9907-c08ae38291e4.json new file mode 100644 index 00000000..f1c49e37 --- /dev/null +++ b/docs/lambda/search-repo/samples/00215aac-e0af-def2-9907-c08ae38291e4.json @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-141,60],[-123.8,60],[-123.8,69.7],[-141,69.7],[-141,60]]]},"properties":{"id":"00215aac-e0af-def2-9907-c08ae38291e4","title":{"en":"Surficial Geology Points (1:250k)","fr":"Points de géologie superficielle (1:250 km)"},"description":{"en":"This GIS dataset is a Yukon-wide compilation of surficial geology points derived from 13 published and unpublished 1:250,000 scale surficial geology maps produced by the GSC and YGS. Point features captured include: field station, fossil and sample locations; glacial landforms such as erratics, kames, kettles, drumlins and flutings; permafrost features such as pingos, palsas, patterned ground and thermokarst depressions; and other non-glacial landforms such as landslides and tors.","fr":"Ce jeu de données SIG est une compilation à l'échelle du Yukon de points de géologie superficielle dérivés de 13 cartes géologiques superficielles publiées et inédites à l'échelle 1:250 000 produites par le GSC et le YGS. Les caractéristiques ponctuelles capturées comprennent : les stations de terrain, les sites fossiles et les emplacements d'échantillonnage ; les formes de relief glaciaires telles que les erratiques, les kames, les bouilloires, les drumlins et les cannelures ; les caractéristiques du pergélisol telles que les pingos, les palsas, les dépressions à motifs et thermokarstiques ; et d'autres formes de relief non glaciaires telles que les glissements de terrain et les tores.** Cet élément de métadonnées provenant d’une tierce partie a été traduit à l'aide d'un outil de traduction automatisée (Amazon Translate).**"},"keywords":{"en":"Yukon Geological Survey, Government information, ","fr":"Relevé géologique du Yukon, Information gouvernementale, "},"topicCategory":"geoscientificInformation","date":{"published":{"text":"publication; publication","date":"2019-05-08"},"created":{"text":"creation; création","date":"2019-05-08"}},"spatialRepresentation":"vector; vecteur","type":"dataset; jeuDonnées","geometry":"POLYGON((-141 60, -123.8 60, -123.8 69.7, -141 69.7, -141 60))","temporalExtent":{"begin":"1969-12-31","end":null},"refSys":"unknown","refSys_version":"unknown","status":"onGoing; enContinue","maintenance":"irregular; irrégulier","metadataStandard":{"en":"North American Profile of ISO 19115:2003 - Geographic information - Metadata","fr":"Profil nord-américain de la norme ISO 19115:2003 - Information géographique - Métadonnées"},"metadataStandardVersion":"HNAP ISO:19115 - 2003","graphicOverview":[{"overviewFileName":"https://raw.githubusercontent.com/federal-geospatial-platform/fgp-metadata-proxy/master/images/resampled/yk/YukonLogo.png","overviewFileDescription":null,"overviewFileTupe":null}],"distributionFormat_name":null,"distributionFormat_format":null,"useLimits":{"en":"Open Government Licence - Yukon (https://open.yukon.ca/open-government-licence-yukon)","fr":"Licence du gouvernement ouvert - Yukon (https://open.yukon.ca/fr/gouvernement-ouvert-licence-du-yukon)"},"accessConstraints":"license; licence","otherConstraints":{"en":null,"fr":null},"dateStamp":null,"dataSetURI":null,"locale":{"language":"French; Français","country":"Canada; Canada","encoding":"utf8; utf8"},"language":"eng; CAN","characterSet":"utf8; utf8","environmentDescription":null,"supplementalInformation":{"en":null,"fr":null},"contact":[{"individual":null,"position":{"en":null,"fr":null},"organisation":{"en":"Government of Yukon; Government of Yukon; Yukon Geological Survey","fr":"Gouvernement du Yukon; Gouvernement du Yukon; Yukon Geological Survey"},"telephone":{"en":null,"fr":null},"fax":null,"address":{"en":null,"fr":null},"city":null,"pt":{"en":null,"fr":null},"postalcode":null,"country":{"en":null,"fr":null},"email":{"en":"eservices@gov.yk.ca","fr":"eservices@gov.yk.ca"},"onlineResource":{"onlineResource":"https://open.yukon.ca/data/","onlineResource_Name":null,"onlineResource_Protocol":"HTTPS","onlineResource_Description":"This service provides a single point of access to open data produced by the Government of Yukon. All of the data you find here is free to use and re-use for commercial or non-commercial purposes."},"hoursOfService":null,"role":"pointOfContact; contact"}],"credits":[],"cited":[{"individual":null,"position":{"en":null,"fr":null},"organisation":{"en":"Government of Yukon; Government of Yukon; Yukon Geological Survey","fr":"Gouvernement du Yukon; Gouvernement du Yukon; Yukon Geological Survey"},"telephone":{"en":null,"fr":null},"fax":null,"address":{"en":null,"fr":null},"city":null,"pt":{"en":null,"fr":null},"postalcode":null,"country":{"en":null,"fr":null},"email":{"en":"eservices@gov.yk.ca","fr":"eservices@gov.yk.ca"},"onlineResource":{"onlineResource":"https://open.yukon.ca/data/","onlineResource_Name":null,"onlineResource_Protocol":"HTTPS","onlineResource_Description":"This service provides a single point of access to open data produced by the Government of Yukon. All of the data you find here is free to use and re-use for commercial or non-commercial purposes."},"hoursOfService":null,"role":"pointOfContact; contact"}],"distributor":[{"individual":null,"position":{"en":null,"fr":null},"organisation":{"en":"Government of Yukon; Government of Yukon; Yukon Geological Survey","fr":"Gouvernement du Yukon; Gouvernement du Yukon; Yukon Geological Survey"},"telephone":{"en":null,"fr":null},"fax":null,"address":{"en":null,"fr":null},"city":null,"pt":{"en":null,"fr":null},"postalcode":null,"country":{"en":null,"fr":null},"email":{"en":"eservices@gov.yk.ca","fr":"eservices@gov.yk.ca"},"onlineResource":{"onlineResource":"https://open.yukon.ca/data/","onlineResource_Name":null,"onlineResource_Protocol":"HTTPS","onlineResource_Description":"This service provides a single point of access to open data produced by the Government of Yukon. All of the data you find here is free to use and re-use for commercial or non-commercial purposes."},"hoursOfService":null,"role":"pointOfContact; contact"}],"options":[{"url":"https://open.yukon.ca/data/datasets/surficial-geology-points-1250k","protocol":"HTTPS","name":{"en":"Original metadata (https://open.yukon.ca)","fr":"Métadonnées originales (https://open.yukon.ca)"},"description":{"en":"Dataset;HTML;eng","fr":"Données;HTML;eng"}},{"url":"http://ygsftp.gov.yk.ca/YGSIDS/compilations/Surficial_2014_04_08/250k/surficial_compilation_250k_2014_04_08.kmz","protocol":"HTTP","name":{"en":"Google Earth (kmz)","fr":"Google Earth (km)"},"description":{"en":"Dataset;KMZ;eng","fr":"Données;KMZ;eng"}},{"url":"https://ygsftp.gov.yk.ca/ygsids/compilations/surficial_2014_04_08/yukon_surficial_geol_codes.xls","protocol":"HTTPS","name":{"en":"Terrain Classification System Codes","fr":"Codes du système de classification des terrains"},"description":{"en":"Dataset;XLS;eng","fr":"Données;XLS;eng"}},{"url":"https://ygsftp.gov.yk.ca/ygsids/compilations/surficial_2014_04_08/250k/surficial_points_compilation_250k.gdb_2014_04_08.zip","protocol":"HTTPS","name":{"en":"Geodatabase","fr":"Géodatabase"},"description":{"en":"Dataset;ZIP;eng","fr":"Données;ZIP;eng"}},{"url":"https://data.geology.gov.yk.ca/compilation/downloadproduct/39355","protocol":"HTTPS","name":{"en":"Yukon Digital Surficial Geology Compilation Info","fr":"Information sur la compilation numérique de géologie superficielle du Yukon"},"description":{"en":"Dataset;other;eng","fr":"Données;other;eng"}}]}}]} \ No newline at end of file diff --git a/docs/lambda/search-repo/samples/00354f07-4b7b-4a4b-82be-748e05eb561e.json b/docs/lambda/search-repo/samples/00354f07-4b7b-4a4b-82be-748e05eb561e.json new file mode 100644 index 00000000..608cbcec --- /dev/null +++ b/docs/lambda/search-repo/samples/00354f07-4b7b-4a4b-82be-748e05eb561e.json @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-121.259662,54.369937],[-115.903654,54.369937],[-115.903654,56.561277],[-121.259662,56.561277],[-121.259662,54.369937]]]},"properties":{"id":"00354f07-4b7b-4a4b-82be-748e05eb561e","title":{"en":"Permian Structure (GIS data, polygon features)","fr":"Structure permienne (données SIG, entités surfaciques)"},"description":{"en":"The Geological Atlas of the Western Canada Sedimentary Basin was designed primarily as a reference volume documenting the subsurface geology of the Western Canada Sedimentary Basin. This GIS dataset is one of a collection of shapefiles representing part of Chapter 15 of the Atlas, Permian Strata of the Western Canada Sedimentary Basin, Figure 13, Permian Structure. Shapefiles were produced from archived digital files created by the Alberta Geological Survey in the mid-1990s, and edited in 2005-06 to correct, attribute and consolidate the data into single files by feature type and by figure.","fr":"L'Atlas géologique du bassin sédimentaire de l'Ouest canadien a été conçu principalement comme un volume de référence documentant la géologie souterraine du bassin sédimentaire de l'Ouest canadien. Cet ensemble de données SIG fait partie d'une collection de fichiers de formes représentant une partie du chapitre 15 de l'Atlas, Strates permienne du bassin sédimentaire de l'Ouest canadien, figure 13, Structure permienne. Les fichiers de formes ont été produits à partir de fichiers numériques archivés créés par l'Alberta Geological Survey au milieu des années 1990 et modifiés en 2005-2006 pour corriger, attribuer et consolider les données dans des fichiers uniques par type d'entité et par figure.** Cet élément de métadonnées provenant d’une tierce partie a été traduit à l'aide d'un outil de traduction automatisée (Amazon Translate).**"},"keywords":{"en":"geology, gis data, western canada sedimentary basin, 83k, 83l, 83m, 83n, 84c, 84d, alberta, canada, western canada, Government information, ","fr":"géologie, données SIG, bassin sédimentaire de l'ouest canadien, 83k, 83l, 83m, 83n, 84c, 84d, Alberta, Canada, l'ouest du canada, Information gouvernementale, "},"topicCategory":"geoscientificInformation","date":{"published":{"text":"publication; publication","date":"2008-01-01"},"created":{"text":"creation; création","date":"0001-01-01"}},"spatialRepresentation":"vector; vecteur","type":"dataset; jeuDonnées","geometry":"POLYGON((-121.259662 54.369937, -115.903654 54.369937, -115.903654 56.561277, -121.259662 56.561277, -121.259662 54.369937))","temporalExtent":{"begin":"0001-01-01","end":"2008-01-01"},"refSys":"EPSG:4269","refSys_version":"8.4.1","status":"completed; complété","maintenance":"unknown; inconnu","metadataStandard":{"en":"North American Profile of ISO 19115:2003 - Geographic information - Metadata","fr":"Profil nord-américain de la norme ISO 19115:2003 - Information géographique - Métadonnées"},"metadataStandardVersion":"ISO 19115-1.1","graphicOverview":[{"overviewFileName":"https://raw.githubusercontent.com/federal-geospatial-platform/fgp-metadata-proxy/master/images/resampled/alberta.png","overviewFileDescription":null,"overviewFileTupe":null}],"distributionFormat_name":null,"distributionFormat_format":null,"useLimits":{"en":"Open Government Licence - Alberta (https://open.alberta.ca/licence)","fr":"Licence du gouvernement ouvert - Alberta (https://open.alberta.ca/licence)"},"accessConstraints":"license; licence","otherConstraints":{"en":null,"fr":null},"dateStamp":null,"dataSetURI":null,"locale":{"language":"French; Français","country":"Canada; Canada","encoding":"utf8; utf8"},"language":"eng; CAN","characterSet":"utf8; utf8","environmentDescription":null,"supplementalInformation":{"en":null,"fr":null},"contact":[{"individual":"Alberta Geological Survey","position":{"en":"AGS Information Manager","fr":"AGS Information Manager"},"organisation":{"en":"Government of Alberta; Government of Alberta; Alberta Geological Survey","fr":"Gouvernement de l'Alberta; Gouvernement de l'Alberta; Commission géologique de l'Alberta"},"telephone":{"en":"(780) 638-4491","fr":"(780) 638-4491"},"fax":"(780) 422-1918","address":{"en":"Alberta Energy Regulator 4th Floor, Twin Atria Building 4999-98 Avenue NW","fr":"Alberta Energy Regulator 4th Floor, Twin Atria Building 4999-98 Avenue NW"},"city":"Edmonton","pt":{"en":"Alberta","fr":"Alberta"},"postalcode":"T6B 2X3","country":{"en":"Canada","fr":"Canada"},"email":{"en":"AGS-Info@aer.ca","fr":"AGS-Info@aer.ca"},"onlineResource":{"onlineResource":"https://geodiscover.alberta.ca/geoportal/","onlineResource_Name":null,"onlineResource_Protocol":"HTTPS","onlineResource_Description":"GeoDiscover Alberta provides enhanced details regarding Alberta's geospatial data."},"hoursOfService":null,"role":"pointOfContact; contact"}],"credits":[],"cited":[{"individual":"Alberta Geological Survey","position":{"en":"AGS Information Manager","fr":"AGS Information Manager"},"organisation":{"en":"Government of Alberta; Government of Alberta; Alberta Geological Survey","fr":"Gouvernement de l'Alberta; Gouvernement de l'Alberta; Commission géologique de l'Alberta"},"telephone":{"en":"(780) 638-4491","fr":"(780) 638-4491"},"fax":"(780) 422-1918","address":{"en":"Alberta Energy Regulator 4th Floor, Twin Atria Building 4999-98 Avenue NW","fr":"Alberta Energy Regulator 4th Floor, Twin Atria Building 4999-98 Avenue NW"},"city":"Edmonton","pt":{"en":"Alberta","fr":"Alberta"},"postalcode":"T6B 2X3","country":{"en":"Canada","fr":"Canada"},"email":{"en":"AGS-Info@aer.ca","fr":"AGS-Info@aer.ca"},"onlineResource":{"onlineResource":"https://geodiscover.alberta.ca/geoportal/","onlineResource_Name":null,"onlineResource_Protocol":"HTTPS","onlineResource_Description":"GeoDiscover Alberta provides enhanced details regarding Alberta's geospatial data."},"hoursOfService":null,"role":"pointOfContact; contact"}],"distributor":[{"individual":"Alberta Geological Survey","position":{"en":"AGS Information Manager","fr":"AGS Information Manager"},"organisation":{"en":"Government of Alberta; Government of Alberta; Alberta Geological Survey","fr":"Gouvernement de l'Alberta; Gouvernement de l'Alberta; Commission géologique de l'Alberta"},"telephone":{"en":"(780) 638-4491","fr":"(780) 638-4491"},"fax":"(780) 422-1918","address":{"en":"Alberta Energy Regulator 4th Floor, Twin Atria Building 4999-98 Avenue NW","fr":"Alberta Energy Regulator 4th Floor, Twin Atria Building 4999-98 Avenue NW"},"city":"Edmonton","pt":{"en":null,"fr":"Alberta"},"postalcode":"T6B 2X3","country":{"en":"Canada","fr":"Canada"},"email":{"en":"AGS-Info@aer.ca","fr":"AGS-Info@aer.ca"},"onlineResource":{"onlineResource":"https://geodiscover.alberta.ca/geoportal/","onlineResource_Name":null,"onlineResource_Protocol":"HTTPS","onlineResource_Description":"GeoDiscover Alberta provides enhanced details regarding Alberta's geospatial data."},"hoursOfService":null,"role":"pointOfContact; contact"}],"options":[{"url":"https://geodiscover.alberta.ca/geoportal/rest/metadata/item/00354f074b7b4a4b82be748e05eb561e/html","protocol":"HTTPS","name":{"en":"Alberta Geoportal","fr":"Géoportail de l'Alberta"},"description":{"en":"Dataset;HTML;eng","fr":"Données;HTML;fra"}},{"url":"https://static.ags.aer.ca/files/document/DIG/DIG_2008_0190.zip","protocol":"HTTPS","name":{"en":"DIG_2008_0190.zip","fr":"DIG_2008_0190.zip"},"description":{"en":"Dataset;SHP;eng","fr":"Données;SHP;fra"}}]}}]} \ No newline at end of file diff --git a/docs/lambda/semantic-search/Invoke-sagemaker-pretrain-20240508-2200.zip b/docs/lambda/semantic-search/Invoke-sagemaker-pretrain-20240508-2200.zip new file mode 100644 index 00000000..7d237585 Binary files /dev/null and b/docs/lambda/semantic-search/Invoke-sagemaker-pretrain-20240508-2200.zip differ diff --git a/docs/lambda/stac-to-geocore/stac-to-geocore-20230330-0000.zip b/docs/lambda/stac-to-geocore/stac-to-geocore-20230330-0000.zip new file mode 100644 index 00000000..4613029f Binary files /dev/null and b/docs/lambda/stac-to-geocore/stac-to-geocore-20230330-0000.zip differ diff --git a/docs/lambda/stac-to-geocore/stac-to-geocore-20230605-1500.zip b/docs/lambda/stac-to-geocore/stac-to-geocore-20230605-1500.zip new file mode 100644 index 00000000..8fe11f67 Binary files /dev/null and b/docs/lambda/stac-to-geocore/stac-to-geocore-20230605-1500.zip differ diff --git a/docs/lambda/stac-to-geocore/stac-to-geocore-20231019-1200.zip b/docs/lambda/stac-to-geocore/stac-to-geocore-20231019-1200.zip new file mode 100644 index 00000000..f0870d25 Binary files /dev/null and b/docs/lambda/stac-to-geocore/stac-to-geocore-20231019-1200.zip differ diff --git a/docs/lambda/stac-to-geocore/stac-to-geocore-20240628-1430.zip b/docs/lambda/stac-to-geocore/stac-to-geocore-20240628-1430.zip new file mode 100644 index 00000000..49dafae0 Binary files /dev/null and b/docs/lambda/stac-to-geocore/stac-to-geocore-20240628-1430.zip differ diff --git a/docs/lambda/viewer-config-service/viewer-config-service-20231218-0800.zip b/docs/lambda/viewer-config-service/viewer-config-service-20231218-0800.zip new file mode 100644 index 00000000..7b645e61 Binary files /dev/null and b/docs/lambda/viewer-config-service/viewer-config-service-20231218-0800.zip differ diff --git a/docs/lambda/viewer-config-service/viewer-config-service-20231218-1200.zip b/docs/lambda/viewer-config-service/viewer-config-service-20231218-1200.zip new file mode 100644 index 00000000..8403f071 Binary files /dev/null and b/docs/lambda/viewer-config-service/viewer-config-service-20231218-1200.zip differ diff --git a/docs/lambda/viewer-config-service/viewer-config-service-20240924-1200.zip b/docs/lambda/viewer-config-service/viewer-config-service-20240924-1200.zip new file mode 100644 index 00000000..55c02dd2 Binary files /dev/null and b/docs/lambda/viewer-config-service/viewer-config-service-20240924-1200.zip differ