Skip to content

Commit

Permalink
Merge pull request #5503 from ministryofjustice/feature/ap-ingestion-…
Browse files Browse the repository at this point in the history
…ingress

🚚 Add egress functionality to Analytical Platform Ingestion
  • Loading branch information
Jacob Woffenden authored Apr 2, 2024
2 parents 4a1d322 + 4d24823 commit 4d3d143
Show file tree
Hide file tree
Showing 15 changed files with 220 additions and 51 deletions.
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
module "transfer_structured_logs" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/cloudwatch/aws//modules/log-group"
version = "5.3.1"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
#### This file can be used to store data specific to the member account ####
data "aws_availability_zones" "available" {}
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,14 @@ locals {
target_buckets = ["dev-ingestion-testing"]

/* Transfer Server */
transfer_server_hostname = "sftp.development.ingestion.analytical-platform.service.justice.gov.uk"
transfer_server_sftp_users = {
"jacobwoffenden" = {
ssh_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+3qaLVtn6Pd+DasWHhIOBoXEEhF9GZAG+DYfJBeySS Ministry of Justice"
cidr_blocks = ["90.246.52.170/32"]
},
"garyhenderson" = {
ssh_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID2lrI7AhZ9Sy/JAVDfPPEkCZawuuVJ7MHg6NNAwYImb"
cidr_blocks = ["154.47.111.68/32"]
transfer_server_hostname = "sftp.development.ingestion.analytical-platform.service.justice.gov.uk"
transfer_server_sftp_users = {}
transfer_server_sftp_users_with_egress = {
"jacobwoffenden-egress" = {
ssh_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+3qaLVtn6Pd+DasWHhIOBoXEEhF9GZAG+DYfJBeySS Ministry of Justice"
cidr_blocks = ["90.246.52.170/32"]
egress_bucket = module.bold_egress_bucket.s3_bucket_id
egress_bucket_kms_key = module.s3_bold_egress_kms.key_arn
}
}
}
Expand All @@ -51,17 +50,9 @@ locals {
target_buckets = ["dev-ingestion-testing"]

/* Transfer Server */
transfer_server_hostname = "sftp.ingestion.analytical-platform.service.justice.gov.uk"
transfer_server_sftp_users = {
"jacobwoffenden" = {
ssh_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+3qaLVtn6Pd+DasWHhIOBoXEEhF9GZAG+DYfJBeySS Ministry of Justice"
cidr_blocks = ["90.246.52.170/32"]
},
"garyhenderson" = {
ssh_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID2lrI7AhZ9Sy/JAVDfPPEkCZawuuVJ7MHg6NNAwYImb"
cidr_blocks = ["154.47.111.68/32"]
}
}
transfer_server_hostname = "sftp.ingestion.analytical-platform.service.justice.gov.uk"
transfer_server_sftp_users = {}
transfer_server_sftp_users_with_egress = {}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
module "transfer_logs_kms" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/kms/aws"
version = "2.2.1"

Expand Down Expand Up @@ -39,6 +40,7 @@ module "transfer_logs_kms" {

module "s3_landing_kms" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/kms/aws"
version = "2.2.1"

Expand All @@ -51,6 +53,7 @@ module "s3_landing_kms" {

module "s3_processed_kms" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/kms/aws"
version = "2.2.1"

Expand All @@ -63,6 +66,7 @@ module "s3_processed_kms" {

module "s3_quarantine_kms" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/kms/aws"
version = "2.2.1"

Expand All @@ -75,6 +79,7 @@ module "s3_quarantine_kms" {

module "s3_definitions_kms" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/kms/aws"
version = "2.2.1"

Expand All @@ -87,6 +92,7 @@ module "s3_definitions_kms" {

module "s3_bold_egress_kms" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/kms/aws"
version = "2.2.1"

Expand Down Expand Up @@ -115,6 +121,7 @@ module "s3_bold_egress_kms" {

module "sns_kms" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/kms/aws"
version = "2.2.1"

Expand All @@ -127,6 +134,7 @@ module "sns_kms" {

module "govuk_notify_kms" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/kms/aws"
version = "2.2.1"

Expand All @@ -139,6 +147,7 @@ module "govuk_notify_kms" {

module "supplier_data_kms" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/kms/aws"
version = "2.2.1"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
module "definition_upload_lambda" {
#checkov:skip=CKV_TF_1:Module is from Terraform registry

source = "terraform-aws-modules/lambda/aws"
version = "7.2.1"

Expand Down Expand Up @@ -53,6 +54,7 @@ module "definition_upload_lambda" {

module "scan_lambda" {
#checkov:skip=CKV_TF_1:Module is from Terraform registry

source = "terraform-aws-modules/lambda/aws"
version = "7.2.1"

Expand Down Expand Up @@ -123,6 +125,7 @@ module "scan_lambda" {

module "transfer_lambda" {
#checkov:skip=CKV_TF_1:Module is from Terraform registry

source = "terraform-aws-modules/lambda/aws"
version = "7.2.1"

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
data "aws_iam_policy_document" "this" {
statement {
sid = "AllowKMS"
effect = "Allow"
actions = [
"kms:ReEncrypt*",
"kms:GenerateDataKey*",
"kms:Encrypt",
"kms:DescribeKey",
"kms:Decrypt",
]
resources = [
var.landing_bucket_kms_key,
var.egress_bucket_kms_key
]
}
statement {
sid = "AllowS3ListBucket"
effect = "Allow"
actions = ["s3:ListBucket"]
resources = [
"arn:aws:s3:::${var.landing_bucket}",
"arn:aws:s3:::${var.egress_bucket}"
]
}
statement {
sid = "AllowS3LandingBucketObjectActions"
effect = "Allow"
actions = ["s3:PutObject"]
resources = ["arn:aws:s3:::${var.landing_bucket}/${var.name}/*"]
}
statement {
sid = "AllowS3EgressBucketObjectActions"
effect = "Allow"
actions = [
"s3:GetObject",
"s3:GetObjectAcl",
"s3:GetObjectVersion"
]
resources = ["arn:aws:s3:::${var.egress_bucket}/${var.name}/*"]
}
}

module "policy" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/iam/aws//modules/iam-policy"
version = "5.37.1"

name_prefix = "transfer-user-${var.name}"

policy = data.aws_iam_policy_document.this.json
}

module "role" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role"
version = "5.37.1"

create_role = true

role_name = "transfer-user-${var.name}"
role_requires_mfa = false

trusted_role_services = ["transfer.amazonaws.com"]

custom_role_policy_arns = [module.policy.arn]
}

resource "aws_transfer_user" "this" {
server_id = var.transfer_server
user_name = var.name
role = module.role.iam_role_arn

home_directory_type = "LOGICAL"
home_directory_mappings {
entry = "/upload"
target = "/${var.landing_bucket}/${var.name}"
}

home_directory_mappings {
entry = "/download"
target = "/${var.egress_bucket}/${var.name}"
}
}

resource "aws_transfer_ssh_key" "this" {
server_id = var.transfer_server
user_name = aws_transfer_user.this.user_name
body = var.ssh_key
}

resource "aws_security_group_rule" "this" {
type = "ingress"
from_port = 2222
to_port = 2222
protocol = "tcp"
cidr_blocks = var.cidr_blocks
security_group_id = var.transfer_server_security_group
}

resource "aws_secretsmanager_secret" "this" {
for_each = toset(["technical-contact", "data-contact", "target-bucket"])

name = "ingestion/sftp/${var.name}/${each.key}"
kms_key_id = var.supplier_data_kms_key
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
variable "name" {
type = string
}

variable "ssh_key" {
type = string
}

variable "cidr_blocks" {
type = list(string)
}

variable "transfer_server" {
type = string
}

variable "transfer_server_security_group" {
type = string
}

variable "landing_bucket" {
type = string
}

variable "landing_bucket_kms_key" {
type = string
}

variable "egress_bucket" {
type = string
}

variable "egress_bucket_kms_key" {
type = string
}

variable "supplier_data_kms_key" {
type = string
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ data "aws_iam_policy_document" "this" {
]
resources = [var.landing_bucket_kms_key]
}
# TODO: review the permissions
statement {
sid = "AllowS3ListBucket"
effect = "Allow"
Expand All @@ -21,17 +20,17 @@ data "aws_iam_policy_document" "this" {
"arn:aws:s3:::${var.landing_bucket}/${var.name}/*"
]
}
# TODO: review the permissions
statement {
sid = "AllowS3ObjectActions"
sid = "AllowS3LandingBucketObjectActions"
effect = "Allow"
actions = ["s3:*"]
actions = ["s3:PutObject"]
resources = ["arn:aws:s3:::${var.landing_bucket}/${var.name}/*"]
}
}

module "policy" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/iam/aws//modules/iam-policy"
version = "5.37.1"

Expand All @@ -57,24 +56,10 @@ module "role" {
}

resource "aws_transfer_user" "this" {
server_id = var.transfer_server
user_name = var.name
role = module.role.iam_role_arn

# This doesn't work unless optimised directory is disabled, and that isn't available in Terraform
# home_directory_type = "LOGICAL"
# home_directory_mappings {
# entry = "/upload"
# target = "/${var.landing_bucket}/${var.name}/upload"
# }

# home_directory_mappings {
# entry = "/download"
# target = "/${var.landing_bucket}/${var.name}/download"
# }

# This works
home_directory = "/${var.landing_bucket}/${var.name}" # TODO: do we need an SFTP specific landing bucket?
server_id = var.transfer_server
user_name = var.name
role = module.role.iam_role_arn
home_directory = "/${var.landing_bucket}/${var.name}"
}

resource "aws_transfer_ssh_key" "this" {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
module "observability_platform_tenant" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "ministryofjustice/observability-platform-tenant/aws"
version = "1.0.0"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
module "ingestion_landing_bucket_notification" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/s3-bucket/aws//modules/notification"
version = "4.1.0"

Expand All @@ -14,6 +16,8 @@ module "ingestion_landing_bucket_notification" {
}

module "ingestion_transfer_bucket_notification" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/s3-bucket/aws//modules/notification"
version = "4.1.0"

Expand Down
Loading

0 comments on commit 4d3d143

Please sign in to comment.