Skip to content

Commit

Permalink
merging in latest changes from main
Browse files Browse the repository at this point in the history
richgreen-moj committed Jun 18, 2024
2 parents 6d5aa6b + ae44c4e commit ed0682a
Showing 31 changed files with 524 additions and 122 deletions.
Original file line number Diff line number Diff line change
@@ -27,3 +27,27 @@ resource "helm_release" "actions_runner_mojas_create_a_derived_table" {
)
]
}

resource "helm_release" "actions_runner_mojas_create_a_derived_table_dpr" {
count = terraform.workspace == "analytical-platform-compute-production" ? 1 : 0

/* https://github.com/ministryofjustice/analytical-platform-actions-runner */
name = "actions-runner-mojas-create-a-derived-table-dpr"
repository = "oci://ghcr.io/ministryofjustice/analytical-platform-charts"
version = "2.317.0"
chart = "actions-runner"
namespace = kubernetes_namespace.actions_runners[0].metadata[0].name
values = [
templatefile(
"${path.module}/src/helm/values/actions-runners/create-a-derived-table/values.yml.tftpl",
{
replicaCount = 1
github_organisation = "moj-analytical-services"
github_repository = "create-a-derived-table"
github_token = data.aws_secretsmanager_secret_version.actions_runners_create_a_derived_table[0].secret_string
github_runner_labels = "digital-prison-reporting"
eks_role_arn = "arn:aws:iam::972272129531:role/dpr-data-api-cross-account-role"
}
)
]
}
Original file line number Diff line number Diff line change
@@ -54,7 +54,14 @@ locals {
/* Transfer Server */
transfer_server_hostname = "sftp.ingestion.analytical-platform.service.justice.gov.uk"
transfer_server_sftp_users = {}
transfer_server_sftp_users_with_egress = {}
transfer_server_sftp_users_with_egress = {
"essex-police" = {
ssh_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpNyYzfbqmy4QkViLRyASRqxrEK7G34o3Bc6Jdp8vK555/oBuAUXUxfavtenZnxuxdxsrBYBSCBFcU4+igeXN/nN2kVfaUlt1xBZBCRUaajinhmt+3CLbr8bWmHR/5vL/DhxHH+j/+gDH5A244XN/ybZQvCGX/ilgKiae8s0tiOZD2hmX0fhRTCohQFG/DIu06gqKIyxUQoHBoBJxjzaDvjqioJgqmD9893DN+Gx1KozmaQWHM+0f7iK1UFp8BkdeFBVkj8TOfx60o/EmAjWQ/U+WSHblaXo0nI+LQKZYkW52uTEnfSkbkyvs/vj8E8+vagwYi0noyTVmb5qReSuk1kyuqEP2ycKIaWKt+Z4LnwxHm7KO51SMMeBgpiFHaUTQWXZHYuU2aXVfFIgJkCtHdEjG7Qe2P8K5XU5rG+CrQ/Y9PxPrKQHk+2nox9dLfCWo2Eho1N85z9/rA7A0oNwsHkjWAl3k87lWdpg7y3VNLzqsMNF4M4HjpQV60MH73dUU= essex-police@kpvmshift04app.netr.ecis.police.uk"
cidr_blocks = ["194.74.29.178/32"]
egress_bucket = module.bold_egress_bucket.s3_bucket_id
egress_bucket_kms_key = module.s3_bold_egress_kms.key_arn
}
}
}
}
}
1 change: 1 addition & 0 deletions terraform/environments/apex/application_variables.json
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@
"appscaling_max_capacity": 2,
"ecs_scaling_cpu_threshold": 80,
"ecs_scaling_mem_threshold": 80,
"ec2_instance_warmup_period": 300,
"container_memory_allocation": 1000,
"region": "eu-west-2",
"docker_image_tag": "development",
1 change: 1 addition & 0 deletions terraform/environments/apex/ecs.tf
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@ module "apex-ecs" {
ecs_scaling_cpu_threshold = local.application_data.accounts[local.environment].ecs_scaling_cpu_threshold
ecs_scaling_mem_threshold = local.application_data.accounts[local.environment].ecs_scaling_mem_threshold
ecs_target_capacity = local.ecs_target_capacity
ec2_instance_warmup_period = local.application_data.accounts[local.environment].ec2_instance_warmup_period
log_group_kms_key = aws_kms_key.cloudwatch_logs_key.arn
environment = local.environment

5 changes: 5 additions & 0 deletions terraform/environments/apex/modules/ecs/main.tf
Original file line number Diff line number Diff line change
@@ -502,6 +502,8 @@ resource "aws_appautoscaling_policy" "ecs_target_cpu" {
predefined_metric_type = "ECSServiceAverageCPUUtilization"
}
target_value = var.ecs_scaling_cpu_threshold
scale_in_cooldown = 300
scale_out_cooldown = 300
}
}

@@ -516,6 +518,8 @@ resource "aws_appautoscaling_policy" "ecs_target_memory" {
predefined_metric_type = "ECSServiceAverageMemoryUtilization"
}
target_value = var.ecs_scaling_mem_threshold
scale_in_cooldown = 300
scale_out_cooldown = 300
}
}

@@ -531,6 +535,7 @@ resource "aws_ecs_capacity_provider" "apex" {
# minimum_scaling_step_size = 1
status = "ENABLED"
target_capacity = var.ecs_target_capacity
instance_warmup_period = var.ec2_instance_warmup_period
}
managed_draining = "ENABLED"
}
5 changes: 5 additions & 0 deletions terraform/environments/apex/modules/ecs/variables.tf
Original file line number Diff line number Diff line change
@@ -142,6 +142,11 @@ variable "ecs_target_capacity" {
description = "The target value for the CloudWatch metric used in the Amazon ECS-managed target tracking scaling policy. For example, a value of 100 will result in the Amazon EC2 instances in your Auto Scaling group being completely utilized."
}

variable "ec2_instance_warmup_period" {
type = string
description = "Period of time, in seconds, after a newly launched Amazon EC2 instance can contribute to CloudWatch metrics for Auto Scaling group for the Capacity Provider."
}

variable "lb_tg_arn" {
type = string
description = "Load balancer target group ARN used by ECS service"
1 change: 1 addition & 0 deletions terraform/environments/cdpt-ifs/loadbalancer.tf
Original file line number Diff line number Diff line change
@@ -46,6 +46,7 @@ resource "aws_lb_target_group" "ifs_target_group" {
unhealthy_threshold = "5"
matcher = "200-499"
timeout = "10"
path = "/health"
}
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# Database Read Access
resource "aws_secretsmanager_secret" "dms_audit_endpoint_source" {
name = local.dms_audit_endpoint_source_secret_name
description = "Database Endpoint for Reading Audited Interaction Replication Data"
kms_key_id = var.account_config.kms_keys.general_shared
tags = var.tags
}

data "aws_iam_policy_document" "dms_audit_endpoint_source" {
statement {
sid = "DMSRoleToReadTheSecret"
effect = "Allow"
principals {
type = "AWS"
identifiers = ["arn:aws:iam::${local.delius_account_id}:role/DMSSecretsManagerAccessRole"]
}
actions = ["secretsmanager:GetSecretValue"]
resources = [aws_secretsmanager_secret.dms_audit_endpoint_source.arn]
}
}

resource "aws_secretsmanager_secret_policy" "dms_audit_endpoint_source" {
secret_arn = aws_secretsmanager_secret.dms_audit_endpoint_source.arn
policy = data.aws_iam_policy_document.dms_audit_endpoint_source.json
}


# ASM Read Access
resource "aws_secretsmanager_secret" "dms_asm_endpoint_source" {
name = local.dms_asm_endpoint_source_secret_name
description = "ASM Endpoint"
kms_key_id = var.account_config.kms_keys.general_shared
tags = var.tags
}

data "aws_iam_policy_document" "dms_asm_endpoint_source" {
statement {
sid = "DMSRoleToReadTheSecret"
effect = "Allow"
principals {
type = "AWS"
identifiers = ["arn:aws:iam::${local.delius_account_id}:role/DMSSecretsManagerAccessRole"]
}
actions = ["secretsmanager:GetSecretValue"]
resources = [aws_secretsmanager_secret.dms_asm_endpoint_source.arn]
}
}

resource "aws_secretsmanager_secret_policy" "dms_asm_endpoint_source" {
secret_arn = aws_secretsmanager_secret.dms_asm_endpoint_source.arn
policy = data.aws_iam_policy_document.dms_asm_endpoint_source.json
}


# Database Write Access
resource "aws_secretsmanager_secret" "dms_audit_endpoint_target" {
name = local.dms_audit_endpoint_source_secret_name
description = "Database Endpoint for Writing Audited Interaction Replication Data"
kms_key_id = var.account_config.kms_keys.general_shared
tags = var.tags
}

data "aws_iam_policy_document" "dms_audit_endpoint_target" {
statement {
sid = "DMSRoleToReadTheSecret"
effect = "Allow"
principals {
type = "AWS"
identifiers = ["arn:aws:iam::${local.delius_account_id}:role/DMSSecretsManagerAccessRole"]
}
actions = ["secretsmanager:GetSecretValue"]
resources = [aws_secretsmanager_secret.dms_audit_endpoint_target.arn]
}
}

resource "aws_secretsmanager_secret_policy" "dms_audit_endpoint_target" {
secret_arn = aws_secretsmanager_secret.dms_audit_endpoint_target.arn
policy = data.aws_iam_policy_document.dms_audit_endpoint_target.json
}

Original file line number Diff line number Diff line change
@@ -297,3 +297,56 @@ resource "aws_iam_role_policy_attachment" "OracleEnterpriseManagementSecretsPoli
role = aws_iam_role.EC2OracleEnterpriseManagementSecretsRole.name
policy_arn = aws_iam_policy.OracleEnterpriseManagementSecretsPolicy.arn
}



# new IAM role OEM setup to allow DMS to access secrets manager and kms keys
resource "aws_iam_role" "DMSSecretsManagerAccessRole" {
name = "DMSSecretsManagerAccessRole"

assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": ["dms.eu-west-2.com"]
},
"Action": "sts:AssumeRole"
}
]
}
EOF
}

resource "aws_iam_role_policy_attachment" "dms_allow_kms_keys_access" {
role = aws_iam_role.DMSSecretsManagerAccessRole.name
policy_arn = aws_iam_policy.business_unit_kms_key_access.arn
}

data "aws_iam_policy_document" "DMSSecretsManagerAccessRolePolicyDocument" {
statement {
sid = "DMSSecretsManagerAccessRolePolicyDocument"
effect = "Allow"
actions = [
"secretsmanager:GetSecretValue"
]
resources = [
"arn:aws:secretsmanager:*:*:secret:dms_audit_endpoint_source-*",
"arn:aws:secretsmanager:*:*:secret:dms_asm_endpoint_source-*",
"arn:aws:secretsmanager:*:*:secret:dms_audit_endpoint_target-*"
]
}
}

resource "aws_iam_policy" "DMSSecretsManagerAccessRolePolicy" {
name = "DMSSecretsManagerAccessRolePolicy"
policy = data.aws_iam_policy_document.DMSSecretsManagerAccessRolePolicyDocument.json
}

resource "aws_iam_role_policy_attachment" "DMSSecretsManagerAccessRolePolicy" {
role = aws_iam_role.DMSSecretsManagerAccessRole.name
policy_arn = aws_iam_policy.DMSSecretsManagerAccessRolePolicy.arn
}

Original file line number Diff line number Diff line change
@@ -5,6 +5,12 @@ locals {

application_secret_name = "${local.secret_prefix}-application-passwords"

dms_audit_endpoint_source_secret_name = "${local.secret_prefix}-dms-audit-endpoint-source"

dms_asm_endpoint_source_secret_name = "${local.secret_prefix}-dms-asm-endpoint-source"

dms_audit_endpoint_target_secret_name = "${local.secret_prefix}-dms-audit-endpoint-target"

oem_account_id = var.platform_vars.environment_management.account_ids[join("-", ["hmpps-oem", var.account_info.mp_environment])]

mis_account_id = var.platform_vars.environment_management.account_ids[join("-", ["delius-mis", var.account_info.mp_environment])]
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@
"rule-id": "01",
"rule-name": "all",
"object-locator": {
"schema-name": "dbo",
"schema-name": "%",
"table-name": "%"
},
"rule-action": "include",
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[D_Comments_V2](
[CommentSID] [int] IDENTITY(1,1) NOT NULL,
[VisitID] [int] NULL,
[ActivityID] [uniqueidentifier] NULL,
[Comments] [varchar](4200) NULL,
[CommentType] [varchar](50) NULL
) ON [PRIMARY]
GO
CREATE CLUSTERED INDEX [PK_D_Comments_V2] ON [dbo].[D_Comments_V2]
(
[CommentSID] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
GO
SET ANSI_PADDING ON
GO
CREATE NONCLUSTERED INDEX [I1_D_Comments_V2] ON [dbo].[D_Comments_V2]
(
[VisitID] ASC,
[CommentType] ASC
)
INCLUDE([CommentSID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
GO
SET ANSI_PADDING ON
GO
CREATE NONCLUSTERED INDEX [I2_D_Comments_V2] ON [dbo].[D_Comments_V2]
(
[CommentType] ASC
)
INCLUDE([CommentSID],[ActivityID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
GO
CREATE NONCLUSTERED INDEX [I3_D_Comments_V2] ON [dbo].[D_Comments_V2]
(
[VisitID] ASC,
[ActivityID] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
GO
SET ANSI_PADDING ON
GO
CREATE NONCLUSTERED INDEX [I4_D_Comments_V2] ON [dbo].[D_Comments_V2]
(
[ActivityID] ASC,
[CommentType] ASC
)
INCLUDE([CommentSID]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
GO
EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'CommentSID'
GO
EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'VisitID'
GO
EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Database ID' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'ActivityID'
GO
EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'Free text comments summarising either a phone call or a visit adding detail that isnt captured else where ' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'Comments'
GO
EXEC sys.sp_addextendedproperty @name=N'MS_Description', @value=N'identifies if the comment belongs to either a phone call or a visit' , @level0type=N'SCHEMA',@level0name=N'dbo', @level1type=N'TABLE',@level1name=N'D_Comments_V2', @level2type=N'COLUMN',@level2name=N'CommentType'
GO
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
SET IDENTITY_INSERT g4s_cap_dw.dbo.D_Comments_V2 ON
;

truncate table [g4s_cap_dw].[dbo].[D_Comments_V2];

INSERT INTO g4s_cap_dw.dbo.D_Comments_V2 (CommentSID, VisitID, ActivityID, Comments, CommentType)
SELECT CommentSID, VisitID, ActivityID,
trim(replace(replace(Comments, char(141), ''), char(129), '')) AS Comments,
CommentType
FROM g4s_cap_dw.dbo.D_Comments
;

SET IDENTITY_INSERT g4s_cap_dw.dbo.D_Comments_V2 OFF;


-- TESTING QUERIES --
--
-- SELECT COUNT(*) FROM g4s_cap_dw.dbo.D_Comments_V2; -- 49695569
-- SELECT COUNT(*) FROM g4s_cap_dw.dbo.D_Comments; -- 49695569
--


--
-- SELECT 'D_Comments' AS TableName, Comments
-- FROM g4s_cap_dw.dbo.D_Comments
-- WHERE CommentSID = 26837791
-- UNION
-- SELECT 'D_Comments_V2' AS TableName, Comments
-- FROM g4s_cap_dw.dbo.D_Comments_V2
-- WHERE CommentSID = 26837791
--
Loading

0 comments on commit ed0682a

Please sign in to comment.