From 40608e96a9b7a606152da7b65b7058d59e546f6d Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 11 Oct 2023 12:00:28 +0100 Subject: [PATCH 01/19] add retentionPeriod to required items --- .../v1.1.0/moj_data_product_metadata_spec.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json b/terraform/environments/data-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json index 8af3b4ea8d7..5a3d4f3ec54 100644 --- a/terraform/environments/data-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json +++ b/terraform/environments/data-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json @@ -11,7 +11,8 @@ "dataProductOwnerDisplayName", "email", "status", - "dpiaRequired" + "dpiaRequired", + "retentionPeriod" ], "properties": { "name": { From f7c7c4809cfe19b7bc24857e52f72600c9b1f8b5 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 11 Oct 2023 12:27:31 +0100 Subject: [PATCH 02/19] add apigateway resources for create schema endpoint --- terraform/environments/data-platform/api.tf | 42 ++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf index 1b389b434ea..b822b5627ee 100644 --- a/terraform/environments/data-platform/api.tf +++ b/terraform/environments/data-platform/api.tf @@ -23,16 +23,19 @@ resource "aws_api_gateway_deployment" "deployment" { aws_api_gateway_resource.data_product_table, aws_api_gateway_resource.data_product_table_name, aws_api_gateway_resource.upload_data_for_data_product_table_name, + aws_api_gateway_resource.create_schema_for_data_product_table_name, aws_api_gateway_method.docs, aws_api_gateway_method.get_glue_metadata, aws_api_gateway_method.register_data_product, aws_api_gateway_method.upload_data_for_data_product_table_name, + aws_api_gateway_method.create_schema_for_data_product_table_name, aws_api_gateway_integration.docs_to_lambda, aws_api_gateway_integration.upload_data_for_data_product_table_name_to_lambda, aws_api_gateway_integration.proxy_to_lambda, aws_api_gateway_integration.docs_lambda_root, aws_api_gateway_integration.get_glue_metadata, - aws_api_gateway_integration.register_data_product_to_lambda + aws_api_gateway_integration.register_data_product_to_lambda, + aws_api_gateway_integration.create_schema_for_data_product_table_name_to_lambda, ])) } @@ -150,6 +153,43 @@ resource "aws_api_gateway_integration" "upload_data_for_data_product_table_name_ } } +# /data-product/{data-product-name}/table/{table-name}/schema resource +resource "aws_api_gateway_resource" "create_schema_for_data_product_table_name" { + parent_id = aws_api_gateway_resource.data_product_table_name.id + path_part = "schema" + rest_api_id = aws_api_gateway_rest_api.data_platform.id +} + +# /data-product/{data-product-name}/table/{table-name}/schema POST method +resource "aws_api_gateway_method" "create_schema_for_data_product_table_name" { + authorization = "CUSTOM" + authorizer_id = aws_api_gateway_authorizer.authorizer.id + http_method = "POST" + resource_id = aws_api_gateway_resource.upload_data_for_data_product_table_name.id + rest_api_id = aws_api_gateway_rest_api.data_platform.id + + request_parameters = { + "method.request.header.Authorization" = true, + "method.request.path.data-product-name" = true, + "method.request.path.table-name" = true, + } +} + +# /data-product/{data-product-name}/table/{table-name}/upload lambda integration +resource "aws_api_gateway_integration" "create_schema_for_data_product_table_name_to_lambda" { + http_method = aws_api_gateway_method.create_schema_for_data_product_table_name.http_method + resource_id = aws_api_gateway_resource.create_schema_for_data_product_table_name.id + rest_api_id = aws_api_gateway_rest_api.data_platform.id + integration_http_method = "POST" + type = "AWS_PROXY" + uri = module.data_product_create_schema_lambda.lambda_function_invoke_arn + + request_parameters = { + "integration.request.path.data-product-name" = "method.request.path.data-product-name", + "integration.request.path.table-name" = "method.request.path.table-name", + } +} + # API docs endpoint resource "aws_api_gateway_resource" "docs" { From 5870616d77a04160074f7179279b6e9d22f9a343 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 11 Oct 2023 12:27:53 +0100 Subject: [PATCH 03/19] add create schema lambda --- .../environments/data-platform/lambda.tf | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index d78fe842e3b..7efbb667013 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -264,3 +264,38 @@ module "resync_unprocessed_files_lambda" { }) } + +module "data_product_create_schema_lambda" { + source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 + application_name = "data_product_create_schema" + tags = local.tags + description = "Lambda to create the first version of a json schema file for a data product" + role_name = "data_product_schema_lambda_role_${local.environment}" + policy_json = data.aws_iam_policy_document.iam_policy_document_for_create_schema_lambda.json + function_name = "data_product_create_schema_${local.environment}" + create_role = true + reserved_concurrent_executions = 1 + + image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-create-schema-lambda-ecr-repo:${local.create_schema_version}" + timeout = 600 + tracing_mode = "Active" + memory_size = 128 + + environment_variables = { + ENVIRONMENT = local.environment + BUCKET_NAME = module.s3-bucket.bucket.id + METADATA_BUCKET = module.s3-bucket.bucket.id + LOG_BUCKET = module.s3-bucket.bucket.id + } + + allowed_triggers = { + + AllowExecutionFromAPIGateway = { + action = "lambda:InvokeFunction" + function_name = "data_product_create_metadata_${local.environment}" + principal = "apigateway.amazonaws.com" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.register_data_product.http_method}${aws_api_gateway_resource.register_data_product.path}" + } + } + +} From f605e9330f01f2ac91fc284dd07740b653d1da34 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 11 Oct 2023 12:28:40 +0100 Subject: [PATCH 04/19] add policy for create schema lambda role --- terraform/environments/data-platform/iam.tf | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index d3e09416e36..eef784678e7 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -575,3 +575,32 @@ data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] } } + +data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { + statement { + sid = "ListBucket" + effect = "Allow" + actions = ["s3:ListBucket"] + resources = [module.s3-bucket.bucket.arn, "${module.s3-bucket.bucket.arn}/*"] + } + statement { + sid = "s3LogAccess" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:PutObject", + ] + resources = [ + "${module.s3-bucket.bucket.arn}/logs/*", + "${module.s3-bucket.bucket.arn}/metadata/*", + "${module.s3-bucket.bucket.arn}/data_product_metadata_spec/*", + "${module.s3-bucket.bucket.arn}/data_product_schema_spec/*" + ] + } + statement { + sid = "LambdaLogGroup" + effect = "Allow" + actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] + resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] + } +} From dfd9f5ce762f5b2119f7226af12f5683068aae6e Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 11 Oct 2023 12:29:11 +0100 Subject: [PATCH 05/19] rebase --- .../data-platform/application_variables.auto.tfvars.json | 6 ++++++ terraform/environments/data-platform/locals.tf | 1 + terraform/environments/data-platform/variables.tf | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/terraform/environments/data-platform/application_variables.auto.tfvars.json b/terraform/environments/data-platform/application_variables.auto.tfvars.json index a98e6ff39d1..8b489d4ac89 100644 --- a/terraform/environments/data-platform/application_variables.auto.tfvars.json +++ b/terraform/environments/data-platform/application_variables.auto.tfvars.json @@ -52,5 +52,11 @@ "test": "1.0.2", "preproduction": "1.0.2", "production": "1.0.2" + }, + "create_schema_versions": { + "development": "1.0.0", + "test": "1.0.0", + "preproduction": "1.0.0", + "production": "1.0.0" } } diff --git a/terraform/environments/data-platform/locals.tf b/terraform/environments/data-platform/locals.tf index 66aa77b6b7b..7a389366e6d 100644 --- a/terraform/environments/data-platform/locals.tf +++ b/terraform/environments/data-platform/locals.tf @@ -32,6 +32,7 @@ locals { create_metadata_version = lookup(var.create_metadata_versions, local.environment) resync_unprocessed_files_version = lookup(var.resync_unprocessed_files_versions, local.environment) reload_data_product_version = lookup(var.reload_data_product_versions, local.environment) + create_schema_version = lookup(var.create_schema_versions, local.environment) landing_to_raw_version = lookup(var.landing_to_raw_versions, local.environment) # Environment vars that are used by many lambdas diff --git a/terraform/environments/data-platform/variables.tf b/terraform/environments/data-platform/variables.tf index 7c92f7fc7b2..4666e7b95f5 100644 --- a/terraform/environments/data-platform/variables.tf +++ b/terraform/environments/data-platform/variables.tf @@ -33,3 +33,7 @@ variable "reload_data_product_versions" { variable "landing_to_raw_versions" { type = map(any) } + +variable "create_schema_versions" { + type = map(any) +} From 779c7d9e1cbd6af1bdb5d5ab7e0fdf513cd19ed5 Mon Sep 17 00:00:00 2001 From: Matt <38562764+LavMatt@users.noreply.github.com> Date: Thu, 12 Oct 2023 07:20:28 +0100 Subject: [PATCH 06/19] Update terraform/environments/data-platform/api.tf --- terraform/environments/data-platform/api.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf index b822b5627ee..220fa254045 100644 --- a/terraform/environments/data-platform/api.tf +++ b/terraform/environments/data-platform/api.tf @@ -175,7 +175,7 @@ resource "aws_api_gateway_method" "create_schema_for_data_product_table_name" { } } -# /data-product/{data-product-name}/table/{table-name}/upload lambda integration +# /data-product/{data-product-name}/table/{table-name}/schema lambda integration resource "aws_api_gateway_integration" "create_schema_for_data_product_table_name_to_lambda" { http_method = aws_api_gateway_method.create_schema_for_data_product_table_name.http_method resource_id = aws_api_gateway_resource.create_schema_for_data_product_table_name.id From 32aa286e0721e7211aafad63d27fe55c77d5f4e4 Mon Sep 17 00:00:00 2001 From: Matt <38562764+LavMatt@users.noreply.github.com> Date: Thu, 12 Oct 2023 07:22:25 +0100 Subject: [PATCH 07/19] Update iam.tf --- terraform/environments/data-platform/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index eef784678e7..4b03c087bd3 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -581,7 +581,7 @@ data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { sid = "ListBucket" effect = "Allow" actions = ["s3:ListBucket"] - resources = [module.s3-bucket.bucket.arn, "${module.s3-bucket.bucket.arn}/*"] + resources = [module.s3-bucket.bucket.arn] } statement { sid = "s3LogAccess" From ec58a366592081274d84a0922c83a12924dc9a3b Mon Sep 17 00:00:00 2001 From: Matt <38562764+LavMatt@users.noreply.github.com> Date: Thu, 12 Oct 2023 11:45:14 +0100 Subject: [PATCH 08/19] Update moj_data_product_table_spec.json --- .../v1.0.0/moj_data_product_table_spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json index bc0e03869fa..4cf09020656 100644 --- a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json +++ b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json @@ -158,6 +158,6 @@ } } }, - "additionalProperties": true + "additionalProperties": false } From e9cf249b44a514095df4f251235546c825ba8bc2 Mon Sep 17 00:00:00 2001 From: Matt <38562764+LavMatt@users.noreply.github.com> Date: Thu, 12 Oct 2023 11:59:49 +0100 Subject: [PATCH 09/19] Update moj_data_product_table_spec.json --- .../v1.0.0/moj_data_product_table_spec.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json index 4cf09020656..d8675cbfb27 100644 --- a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json +++ b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json @@ -18,16 +18,14 @@ "type": "string", "description": "The name of a column within your data.", "pattern": "^[a-z0-9_]+$", - "example": "my_data_column", "maxLength": 255 }, "description": { "type": "string", "description": "A description of the column that will feed the data catalogue.", - "example": "column contains the primary key and relates to a personal identifier autogrenerated by the source database" } }, - "additionalProperties": true + "additionalProperties": false }, "int_types": { "allOf": [ From 583f39b65833a0ff14d42a46b0242706cd7af8b0 Mon Sep 17 00:00:00 2001 From: Matt <38562764+LavMatt@users.noreply.github.com> Date: Thu, 12 Oct 2023 12:00:48 +0100 Subject: [PATCH 10/19] Update moj_data_product_table_spec.json --- .../v1.0.0/moj_data_product_table_spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json index d8675cbfb27..4884758c401 100644 --- a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json +++ b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json @@ -22,7 +22,7 @@ }, "description": { "type": "string", - "description": "A description of the column that will feed the data catalogue.", + "description": "A description of the column that will feed the data catalogue." } }, "additionalProperties": false From 48e578c2c26274b126329072f7d4b857042bb095 Mon Sep 17 00:00:00 2001 From: Matt <38562764+LavMatt@users.noreply.github.com> Date: Thu, 12 Oct 2023 18:48:29 +0100 Subject: [PATCH 11/19] Update moj_data_product_table_spec.json --- .../v1.0.0/moj_data_product_table_spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json index 4884758c401..f27b5dfb2d4 100644 --- a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json +++ b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json @@ -18,7 +18,7 @@ "type": "string", "description": "The name of a column within your data.", "pattern": "^[a-z0-9_]+$", - "maxLength": 255 + "maxLength": 128 }, "description": { "type": "string", From 34bfc500c858d6bfba0ef51a8c1902e3d20f17de Mon Sep 17 00:00:00 2001 From: Matt <38562764+LavMatt@users.noreply.github.com> Date: Thu, 12 Oct 2023 19:00:24 +0100 Subject: [PATCH 12/19] Update moj_data_product_table_spec.json --- .../v1.0.0/moj_data_product_table_spec.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json index f27b5dfb2d4..72d16823a1c 100644 --- a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json +++ b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json @@ -4,6 +4,7 @@ "description": "Specification for MoJ Data Platform Data Product tables schemas for registering tables in the glue catalog", "type": "object", "required": [ + "tableDescription", "columns" ], "definitions": { @@ -128,10 +129,12 @@ "properties": { "tableDescription": { "type": "string", - "description": "A description of the data contained within the table" + "description": "A description of the data contained within the table", + "example": "this table contains example data for an example data product." }, "columns": { "type": "array", + "description": "list of objects which relate to columns in your data, each list item will contain, a name of the column, data type of the column and description of the column", "items": { "oneOf": [ { From 9bac43eaec14c4dd55f33f7cb4a50ca924bc6493 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Fri, 13 Oct 2023 09:40:46 +0100 Subject: [PATCH 13/19] policies align to new buckets --- terraform/environments/data-platform/iam.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index 4b03c087bd3..755a67d2181 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -581,7 +581,7 @@ data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { sid = "ListBucket" effect = "Allow" actions = ["s3:ListBucket"] - resources = [module.s3-bucket.bucket.arn] + resources = [module.logs_s3_bucket.bucket.arn,module.metadata_s3_bucket.arn] } statement { sid = "s3LogAccess" @@ -591,10 +591,9 @@ data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { "s3:PutObject", ] resources = [ - "${module.s3-bucket.bucket.arn}/logs/*", - "${module.s3-bucket.bucket.arn}/metadata/*", - "${module.s3-bucket.bucket.arn}/data_product_metadata_spec/*", - "${module.s3-bucket.bucket.arn}/data_product_schema_spec/*" + "${module.logs_s3_bucket.bucket.arn}/logs/*", + "${module.metadata_s3_bucket.arn}/*", + ] } statement { From 0f3135448f75a0319df6ac96de73d82fe7607979 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Fri, 13 Oct 2023 09:52:36 +0100 Subject: [PATCH 14/19] add in new buckets --- terraform/environments/data-platform/iam.tf | 4 ++-- terraform/environments/data-platform/lambda.tf | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index 755a67d2181..01d4e99466a 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -581,7 +581,7 @@ data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { sid = "ListBucket" effect = "Allow" actions = ["s3:ListBucket"] - resources = [module.logs_s3_bucket.bucket.arn,module.metadata_s3_bucket.arn] + resources = [module.logs_s3_bucket.bucket.arn,module.metadata_s3_bucket.bucket.arn] } statement { sid = "s3LogAccess" @@ -592,7 +592,7 @@ data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { ] resources = [ "${module.logs_s3_bucket.bucket.arn}/logs/*", - "${module.metadata_s3_bucket.arn}/*", + "${module.metadata_s3_bucket.bucket.arn}/*", ] } diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index 7efbb667013..206267f4866 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -281,20 +281,14 @@ module "data_product_create_schema_lambda" { tracing_mode = "Active" memory_size = 128 - environment_variables = { - ENVIRONMENT = local.environment - BUCKET_NAME = module.s3-bucket.bucket.id - METADATA_BUCKET = module.s3-bucket.bucket.id - LOG_BUCKET = module.s3-bucket.bucket.id - } - + environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) allowed_triggers = { AllowExecutionFromAPIGateway = { action = "lambda:InvokeFunction" function_name = "data_product_create_metadata_${local.environment}" principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.register_data_product.http_method}${aws_api_gateway_resource.register_data_product.path}" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.create_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.create_schema_for_data_product_table_name.path}" } } From a125fc5fa102f5d5c1ddf3a81ba972d9ccba87c6 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Fri, 13 Oct 2023 10:02:07 +0100 Subject: [PATCH 15/19] load table schema json schema object to s3 --- terraform/environments/data-platform/s3.tf | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/s3.tf b/terraform/environments/data-platform/s3.tf index 395e0bede69..72107cc10a6 100644 --- a/terraform/environments/data-platform/s3.tf +++ b/terraform/environments/data-platform/s3.tf @@ -316,7 +316,7 @@ resource "aws_s3_bucket_notification" "landing_bucket_notification" { } # load the json schema for data product metadata -resource "aws_s3_object" "object" { +resource "aws_s3_object" "object_metadata" { bucket = module.metadata_s3_bucket.bucket.id key = "data_product_metadata_spec/v1.1.0/moj_data_product_metadata_spec.json" source = "data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json" @@ -324,3 +324,14 @@ resource "aws_s3_object" "object" { acl = "bucket-owner-full-control" server_side_encryption = "AES256" } + +# load the json schema for data product schema +resource "aws_s3_object" "object_schema" { + bucket = module.metadata_s3_bucket.bucket.id + key = "data_product_schema_spec/v1.0.0/moj_data_product_schema_spec.json" + source = "data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json" + etag = filemd5("data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json") + acl = "bucket-owner-full-control" + server_side_encryption = "AES256" +} +# data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json \ No newline at end of file From 5589dd8b3492912a288918a4219c60952b6b4180 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Fri, 13 Oct 2023 12:05:04 +0100 Subject: [PATCH 16/19] fix some minor issues --- terraform/environments/data-platform/api.tf | 2 +- terraform/environments/data-platform/s3.tf | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf index 220fa254045..022d6ca1444 100644 --- a/terraform/environments/data-platform/api.tf +++ b/terraform/environments/data-platform/api.tf @@ -165,7 +165,7 @@ resource "aws_api_gateway_method" "create_schema_for_data_product_table_name" { authorization = "CUSTOM" authorizer_id = aws_api_gateway_authorizer.authorizer.id http_method = "POST" - resource_id = aws_api_gateway_resource.upload_data_for_data_product_table_name.id + resource_id = aws_api_gateway_resource.create_schema_for_data_product_table_name.id rest_api_id = aws_api_gateway_rest_api.data_platform.id request_parameters = { diff --git a/terraform/environments/data-platform/s3.tf b/terraform/environments/data-platform/s3.tf index 72107cc10a6..3ad35d7c135 100644 --- a/terraform/environments/data-platform/s3.tf +++ b/terraform/environments/data-platform/s3.tf @@ -334,4 +334,3 @@ resource "aws_s3_object" "object_schema" { acl = "bucket-owner-full-control" server_side_encryption = "AES256" } -# data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json \ No newline at end of file From c1355a138a38ba708096c606bb4bdf91e022d240 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Fri, 13 Oct 2023 12:54:36 +0100 Subject: [PATCH 17/19] fix policy attachement in schema lambda --- terraform/environments/data-platform/iam.tf | 14 +++++--------- terraform/environments/data-platform/lambda.tf | 1 + 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index 01d4e99466a..3a26b32e73c 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -479,8 +479,7 @@ data "aws_iam_policy_document" "iam_policy_document_for_create_metadata_lambda" effect = "Allow" actions = ["s3:GetObject", "s3:PutObject"] resources = [ - "${module.metadata_s3_bucket.bucket.arn}/metadata/*", - "${module.metadata_s3_bucket.bucket.arn}/data_product_metadata_spec/*" + "${module.metadata_s3_bucket.bucket.arn}/*" ] } @@ -577,21 +576,18 @@ data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files } data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { + source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] statement { sid = "ListBucket" effect = "Allow" actions = ["s3:ListBucket"] - resources = [module.logs_s3_bucket.bucket.arn,module.metadata_s3_bucket.bucket.arn] + resources = [module.metadata_s3_bucket.bucket.arn] } statement { - sid = "s3LogAccess" + sid = "s3MetadataWrite" effect = "Allow" - actions = [ - "s3:GetObject", - "s3:PutObject", - ] + actions = ["s3:PutObject"] resources = [ - "${module.logs_s3_bucket.bucket.arn}/logs/*", "${module.metadata_s3_bucket.bucket.arn}/*", ] diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index 206267f4866..e8b8634553d 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -272,6 +272,7 @@ module "data_product_create_schema_lambda" { description = "Lambda to create the first version of a json schema file for a data product" role_name = "data_product_schema_lambda_role_${local.environment}" policy_json = data.aws_iam_policy_document.iam_policy_document_for_create_schema_lambda.json + policy_json_attached = true function_name = "data_product_create_schema_${local.environment}" create_role = true reserved_concurrent_executions = 1 From 72f55111baf080a3208ab16e740ff0d64080ed79 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Fri, 13 Oct 2023 13:41:16 +0100 Subject: [PATCH 18/19] fix log permissions, add listbucket --- terraform/environments/data-platform/iam.tf | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index 3a26b32e73c..be8f7b10fad 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -7,6 +7,7 @@ data "aws_iam_policy_document" "log_to_bucket" { actions = [ "s3:GetObject", "s3:PutObject", + "s3:ListBucket" ] resources = [ "${module.logs_s3_bucket.bucket.arn}", @@ -577,12 +578,6 @@ data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] - statement { - sid = "ListBucket" - effect = "Allow" - actions = ["s3:ListBucket"] - resources = [module.metadata_s3_bucket.bucket.arn] - } statement { sid = "s3MetadataWrite" effect = "Allow" From b82020815813599f23bf8b51f235b397df01873e Mon Sep 17 00:00:00 2001 From: LavMatt Date: Fri, 13 Oct 2023 13:44:35 +0100 Subject: [PATCH 19/19] fix schema so does not incorrectly fail validation --- .../v1.0.0/moj_data_product_table_spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json index 72d16823a1c..402aa09f09f 100644 --- a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json +++ b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json @@ -26,7 +26,7 @@ "description": "A description of the column that will feed the data catalogue." } }, - "additionalProperties": false + "additionalProperties": true }, "int_types": { "allOf": [