Merge pull request #3629 from ministryofjustice/dpl-1194-add-infra-fo…

…r-create-schema-endpoint Dpl 1194 add infra for create schema endpoint
ministryofjustice · Oct 16, 2023 · be93ac4 · be93ac4
2 parents 24aa031 + b820208
commit be93ac4
Show file tree

Hide file tree

Showing 9 changed files with 123 additions and 11 deletions.
diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf
@@ -23,16 +23,19 @@ resource "aws_api_gateway_deployment" "deployment" {
       aws_api_gateway_resource.data_product_table,
       aws_api_gateway_resource.data_product_table_name,
       aws_api_gateway_resource.upload_data_for_data_product_table_name,
+      aws_api_gateway_resource.create_schema_for_data_product_table_name,
       aws_api_gateway_method.docs,
       aws_api_gateway_method.get_glue_metadata,
       aws_api_gateway_method.register_data_product,
       aws_api_gateway_method.upload_data_for_data_product_table_name,
+      aws_api_gateway_method.create_schema_for_data_product_table_name,
       aws_api_gateway_integration.docs_to_lambda,
       aws_api_gateway_integration.upload_data_for_data_product_table_name_to_lambda,
       aws_api_gateway_integration.proxy_to_lambda,
       aws_api_gateway_integration.docs_lambda_root,
       aws_api_gateway_integration.get_glue_metadata,
-      aws_api_gateway_integration.register_data_product_to_lambda
+      aws_api_gateway_integration.register_data_product_to_lambda,
+      aws_api_gateway_integration.create_schema_for_data_product_table_name_to_lambda,
     ]))
   }
 
@@ -150,6 +153,43 @@ resource "aws_api_gateway_integration" "upload_data_for_data_product_table_name_
   }
 }
 
+# /data-product/{data-product-name}/table/{table-name}/schema resource
+resource "aws_api_gateway_resource" "create_schema_for_data_product_table_name" {
+  parent_id   = aws_api_gateway_resource.data_product_table_name.id
+  path_part   = "schema"
+  rest_api_id = aws_api_gateway_rest_api.data_platform.id
+}
+
+# /data-product/{data-product-name}/table/{table-name}/schema POST method
+resource "aws_api_gateway_method" "create_schema_for_data_product_table_name" {
+  authorization = "CUSTOM"
+  authorizer_id = aws_api_gateway_authorizer.authorizer.id
+  http_method   = "POST"
+  resource_id   = aws_api_gateway_resource.create_schema_for_data_product_table_name.id
+  rest_api_id   = aws_api_gateway_rest_api.data_platform.id
+
+  request_parameters = {
+    "method.request.header.Authorization"   = true,
+    "method.request.path.data-product-name" = true,
+    "method.request.path.table-name"        = true,
+  }
+}
+
+# /data-product/{data-product-name}/table/{table-name}/schema lambda integration
+resource "aws_api_gateway_integration" "create_schema_for_data_product_table_name_to_lambda" {
+  http_method             = aws_api_gateway_method.create_schema_for_data_product_table_name.http_method
+  resource_id             = aws_api_gateway_resource.create_schema_for_data_product_table_name.id
+  rest_api_id             = aws_api_gateway_rest_api.data_platform.id
+  integration_http_method = "POST"
+  type                    = "AWS_PROXY"
+  uri                     = module.data_product_create_schema_lambda.lambda_function_invoke_arn
+
+  request_parameters = {
+    "integration.request.path.data-product-name" = "method.request.path.data-product-name",
+    "integration.request.path.table-name"        = "method.request.path.table-name",
+  }
+}
+
 # API docs endpoint
 
 resource "aws_api_gateway_resource" "docs" {

diff --git a/terraform/environments/data-platform/application_variables.auto.tfvars.json b/terraform/environments/data-platform/application_variables.auto.tfvars.json
@@ -52,5 +52,11 @@
     "test": "1.0.2",
     "preproduction": "1.0.2",
     "production": "1.0.2"
+  },
+  "create_schema_versions": {
+    "development": "1.0.0",
+    "test": "1.0.0",
+    "preproduction": "1.0.0",
+    "production": "1.0.0"
   }
 }
diff --git a/...ata-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json b/...ata-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json
@@ -11,7 +11,8 @@
     "dataProductOwnerDisplayName",
     "email",
     "status",
-    "dpiaRequired"
+    "dpiaRequired",
+    "retentionPeriod"
   ],
   "properties": {
     "name": {

diff --git a/...ta-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json b/...ta-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json
@@ -4,6 +4,7 @@
   "description": "Specification for MoJ Data Platform Data Product tables schemas for registering tables in the glue catalog",
   "type": "object",
   "required": [
+    "tableDescription",
     "columns"
   ],
   "definitions": {
@@ -18,13 +19,11 @@
             "type": "string",
             "description": "The name of a column within your data.",
             "pattern": "^[a-z0-9_]+$",
-            "example": "my_data_column",
-            "maxLength": 255
+            "maxLength": 128
           },
           "description": {
             "type": "string",
-            "description": "A description of the column that will feed the data catalogue.",
-            "example": "column contains the primary key and relates to a personal identifier autogrenerated by the source database"
+            "description": "A description of the column that will feed the data catalogue."
           }
       },
       "additionalProperties": true
@@ -130,10 +129,12 @@
   "properties": {
     "tableDescription": {
       "type": "string",
-      "description": "A description of the data contained within the table"
+      "description": "A description of the data contained within the table",
+      "example": "this table contains example data for an example data product."
     },
     "columns": {
       "type": "array",
+      "description": "list of objects which relate to columns in your data, each list item will contain, a name of the column, data type of the column and description of the column",
       "items": {
         "oneOf": [
           {
@@ -158,6 +159,6 @@
       }
     }
   },
-  "additionalProperties": true
+  "additionalProperties": false
 }
 
diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf
@@ -7,6 +7,7 @@ data "aws_iam_policy_document" "log_to_bucket" {
     actions = [
       "s3:GetObject",
       "s3:PutObject",
+      "s3:ListBucket"
     ]
     resources = [
       "${module.logs_s3_bucket.bucket.arn}",
@@ -479,8 +480,7 @@ data "aws_iam_policy_document" "iam_policy_document_for_create_metadata_lambda"
     effect  = "Allow"
     actions = ["s3:GetObject", "s3:PutObject"]
     resources = [
-      "${module.metadata_s3_bucket.bucket.arn}/metadata/*",
-      "${module.metadata_s3_bucket.bucket.arn}/data_product_metadata_spec/*"
+      "${module.metadata_s3_bucket.bucket.arn}/*"
     ]
   }
 
@@ -575,3 +575,22 @@ data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files
     resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"]
   }
 }
+
+data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" {
+  source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json]
+  statement {
+    sid    = "s3MetadataWrite"
+    effect = "Allow"
+    actions = ["s3:PutObject"]
+    resources = [
+      "${module.metadata_s3_bucket.bucket.arn}/*",
+
+    ]
+  }
+  statement {
+    sid       = "LambdaLogGroup"
+    effect    = "Allow"
+    actions   = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"]
+    resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"]
+  }
+}
diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf
@@ -264,3 +264,33 @@ module "resync_unprocessed_files_lambda" {
   })
 
 }
+
+module "data_product_create_schema_lambda" {
+  source                         = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1
+  application_name               = "data_product_create_schema"
+  tags                           = local.tags
+  description                    = "Lambda to create the first version of a json schema file for a data product"
+  role_name                      = "data_product_schema_lambda_role_${local.environment}"
+  policy_json                    = data.aws_iam_policy_document.iam_policy_document_for_create_schema_lambda.json
+  policy_json_attached           = true
+  function_name                  = "data_product_create_schema_${local.environment}"
+  create_role                    = true
+  reserved_concurrent_executions = 1
+
+  image_uri    = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-create-schema-lambda-ecr-repo:${local.create_schema_version}"
+  timeout      = 600
+  tracing_mode = "Active"
+  memory_size  = 128
+
+  environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars)
+  allowed_triggers = {
+
+    AllowExecutionFromAPIGateway = {
+      action        = "lambda:InvokeFunction"
+      function_name = "data_product_create_metadata_${local.environment}"
+      principal     = "apigateway.amazonaws.com"
+      source_arn    = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.create_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.create_schema_for_data_product_table_name.path}"
+    }
+  }
+
+}
diff --git a/terraform/environments/data-platform/locals.tf b/terraform/environments/data-platform/locals.tf
@@ -32,6 +32,7 @@ locals {
   create_metadata_version          = lookup(var.create_metadata_versions, local.environment)
   resync_unprocessed_files_version = lookup(var.resync_unprocessed_files_versions, local.environment)
   reload_data_product_version      = lookup(var.reload_data_product_versions, local.environment)
+  create_schema_version            = lookup(var.create_schema_versions, local.environment)
   landing_to_raw_version           = lookup(var.landing_to_raw_versions, local.environment)
 
   # Environment vars that are used by many lambdas

diff --git a/terraform/environments/data-platform/s3.tf b/terraform/environments/data-platform/s3.tf
@@ -316,11 +316,21 @@ resource "aws_s3_bucket_notification" "landing_bucket_notification" {
 }
 
 # load the json schema for data product metadata
-resource "aws_s3_object" "object" {
+resource "aws_s3_object" "object_metadata" {
   bucket                 = module.metadata_s3_bucket.bucket.id
   key                    = "data_product_metadata_spec/v1.1.0/moj_data_product_metadata_spec.json"
   source                 = "data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json"
   etag                   = filemd5("data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json")
   acl                    = "bucket-owner-full-control"
   server_side_encryption = "AES256"
 }
+
+# load the json schema for data product schema
+resource "aws_s3_object" "object_schema" {
+  bucket                 = module.metadata_s3_bucket.bucket.id
+  key                    = "data_product_schema_spec/v1.0.0/moj_data_product_schema_spec.json"
+  source                 = "data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json"
+  etag                   = filemd5("data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json")
+  acl                    = "bucket-owner-full-control"
+  server_side_encryption = "AES256"
+}
diff --git a/terraform/environments/data-platform/variables.tf b/terraform/environments/data-platform/variables.tf
@@ -33,3 +33,7 @@ variable "reload_data_product_versions" {
 variable "landing_to_raw_versions" {
   type = map(any)
 }
+
+variable "create_schema_versions" {
+  type = map(any)
+}