🛣️ Create Data Production Transit VPC

.github/dependabot.yml

@@ -55,6 +55,7 @@ updates:
       - "terraform/aws/analytical-platform-data-production/create-a-derived-table"
       - "terraform/aws/analytical-platform-data-production/data-engineering-pipelines"
       - "terraform/aws/analytical-platform-data-production/github-airflow-cjs-dashboard-data"
+      - "terraform/aws/analytical-platform-data-production/hmcts-sdp-transit-direct-connect"
       - "terraform/aws/analytical-platform-data-production/ingestion-egress"
       - "terraform/aws/analytical-platform-data-production/openmetadata"
       - "terraform/aws/analytical-platform-data-production/powerbi-gateway"

.github/workflows/platform-pagerduty-rota-to-slack.yml

@@ -29,7 +29,7 @@ jobs:
 
       - name: Setup Python
         id: setup_python
-        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
         with:
           python-version: 3.9
 

.github/workflows/repository-dependency-review.yml

@@ -22,6 +22,6 @@ jobs:
 
       - name: Dependency review
         id: dependency_review
-        uses: actions/dependency-review-action@72eb03d02c7872a771aacd928f3123ac62ad6d3a # v4.3.3
+        uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4
         with:
           fail-on-severity: critical

.github/workflows/reusable-workflow-terraform.yml

@@ -54,7 +54,7 @@ jobs:
       - name: Checkov
         if: github.ref != 'refs/heads/main'
         id: terraform_static_analysis_checkov
-        uses: bridgecrewio/checkov-action@822daab5b7e499b443dd08e0217886b11a8b71fd # v12.2811.0
+        uses: bridgecrewio/checkov-action@e28bcecbf174dfbefd2829b5e2ded9d5aead6e9e # v12.2821.0
         with:
           directory: ${{ env.working-directory }}
           framework: terraform
@@ -66,7 +66,7 @@ jobs:
       - name: Trivy
         if: github.ref != 'refs/heads/main'
         id: terraform_static_analysis_trivy
-        uses: aquasecurity/trivy-action@7c2007bcb556501da015201bcba5aa14069b74e2 # v0.23.0
+        uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
         with:
           scan-type: config
           scan-ref: ${{ env.working-directory }}

source/documentation/adrs/adr-011-use-lake-formation-for-data-access-management.html.md.erb

@@ -0,0 +1,44 @@
+---
+owner_slack: "#analytical-platform-notifications"
+title: ADR-011 Use Lake Formation for data access management
+last_reviewed_on: 2024-07-05
+review_in: 6 months
+---
+
+# <%= current_page.data.title %>
+
+## Status
+
+✅ Accepted
+
+## Context
+
+We use [IAM](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html) to manage access to data based on resources. However, IAM lacks fine-grained access controls, such as column and row level permissions. Additionally, we are receiving increasing requests to share sensitive data [across accounts](/documentation/adrs/adr-009-use-separate-aws-accounts-for-data.html), where data producers want to control access to their own data. This highlights the need for solutions that allow more detailed access management and data governance.
+
+## Decision
+
+We have chosen to implement AWS [Lake Formation](https://docs.aws.amazon.com/lake-formation/latest/dg/what-is-lake-formation.html) to meet our needs for fine-grained data permissions and robust data governance. While IAM manages resource-based permissions, it does not offer the column and row level controls we require. Lake Formation addresses these gaps by providing fine-grained access management capabilities.
+
+Additionally, Lake Formation supports our growing need to share sensitive data across accounts, enabling data producers to govern their own data. This solution not only enhances security and compliance but also streamlines the process of data sharing and management within our organization.
+
+## Consequences
+
+### General consequences
+
+- We will need to support and maintain a Terraform module for teams to enable and configure Lake Formation
+- Current methods for granting and revoking will need to be reviewed with users
+- We will need to build and maintain a central tag repository to avoid tagging collisions
+- We will still require a solution for unstructured data
+
+### Advantages
+
+- Enables secure data sharing across accounts. Data can stay within the account without the need for exporting or data pipelines which reduces duplication
+- Integration with AWS Identity Center and our existing identity management system
+- Improved data compliance with [security event logging](https://docs.aws.amazon.com/lake-formation/latest/dg/security-event-logging.html) and auditing capabilities
+- We can give data owners direct control over who has access to their data
+- Fine and coarse-grain access control with attributes from users Entra ID profile
+- We can make use of tag based access control [TBAC](https://docs.aws.amazon.com/lake-formation/latest/dg/tag-based-access-control.html) also known as attribute-based access control [ABAC](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction_attribute-based-access-control.html#introduction_attribute-based-access-control_compare-rbac). This reduces the number of access policies and roles
+
+### Disadvantages
+
+- Onboarding of datasets will need more up front work by engineers

source/documentation/adrs/adr-index.html.md.erb

@@ -24,6 +24,7 @@ To understand why we are recording decisions and how we are doing it, please see
 | ADR-008 | 🤔 | [Documentation](/documentation/adrs/adr-008-documentation.html) |
 | ADR-009 | 🤔 | [Use separate AWS accounts for data domains and products](/documentation/adrs/adr-009-use-separate-aws-accounts-for-data.html) |
 | ADR-010 | ✅ | [Use AWS IAM Identity Center customer managed applications for user access](/documentation/adrs/adr-010-use-aws-iam-identity-center-managed-applications-for-user-access.html) |
+| ADR-011 | ✅ | [Use Lake Formation for data access management](/documentation/adrs/adr-011-use-lake-formation-for-data-access-management.html) |
 
 **Statuses:**
 

terraform/auth0/alpha-analytics-moj/terraform.tf

@@ -10,7 +10,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "5.57.0"
+      version = "5.58.0"
     }
     auth0 = {
       source  = "auth0/auth0"

terraform/auth0/dev-analytics-moj/terraform.tf

@@ -10,7 +10,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "5.57.0"
+      version = "5.58.0"
     }
     auth0 = {
       source  = "auth0/auth0"

terraform/aws/analytical-platform-data-engineering-production/10ds/terraform.tf

@@ -10,7 +10,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "5.55.0"
+      version = "5.57.0"
     }
   }
   required_version = "~> 1.5"

terraform/aws/analytical-platform-data-engineering-sandbox-a/airflow-create-a-pipeline/iam-policies.tf

@@ -146,7 +146,7 @@ module "airflow_create_a_pipeline_iam_policy" {
   #checkov:skip=CKV_TF_1:Module is from Terraform registry
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
-  version = "5.40.0"
+  version = "5.41.0"
 
   name_prefix = "github-airflow-create-a-pipeline"
 

terraform/aws/analytical-platform-data-engineering-sandbox-a/airflow-create-a-pipeline/iam-roles.tf

@@ -2,7 +2,7 @@ module "airflow_create_a_pipeline_iam_role" {
   #checkov:skip=CKV_TF_1:Module is from Terraform registry
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-github-oidc-role"
-  version = "5.40.0"
+  version = "5.41.0"
 
   name = "github-airflow-create-a-pipeline"
 

terraform/aws/analytical-platform-data-production/airflow/iam-policies.tf

@@ -11,7 +11,7 @@ module "airflow_analytical_platform_development_iam_policy" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
-  version = "5.40.0"
+  version = "5.41.0"
 
   name = "airflow-analytical-platform-development"
 
@@ -301,7 +301,7 @@ module "airflow_dev_monitoring_iam_policy" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
-  version = "5.40.0"
+  version = "5.41.0"
 
   name = "airflow_dev_monitoring"
 

terraform/aws/analytical-platform-data-production/airflow/iam-roles.tf

@@ -2,7 +2,7 @@ module "airflow_analytical_platform_development_iam_role" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "5.40.0"
+  version = "5.41.0"
 
   create_role = true
 
@@ -91,7 +91,7 @@ module "airflow_dev_monitoring_iam_role" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "5.40.0"
+  version = "5.41.0"
 
   create_role = true
 

terraform/aws/analytical-platform-data-production/airflow/terraform.tf

@@ -10,7 +10,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "5.52.0"
+      version = "5.57.0"
     }
     tls = {
       source  = "hashicorp/tls"