Add IRSA policies for s3 Tables access

analytics/terraform/spark-k8s-operator/README.md

@@ -43,6 +43,7 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/
 |------|------|
 | [aws_eks_access_entry.karpenter_nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource |
 | [aws_iam_policy.grafana](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.s3tables](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.s3tables_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.spark](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_prometheus_workspace.amp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_workspace) | resource |
@@ -62,6 +63,7 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/
 | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source |
 | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
 | [aws_iam_policy_document.grafana](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.s3tables_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.spark_operator](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_session_context.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_session_context) | data source |
 | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
@@ -78,7 +80,7 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/
 | <a name="input_enable_vpc_endpoints"></a> [enable\_vpc\_endpoints](#input\_enable\_vpc\_endpoints) | Enable VPC Endpoints | `bool` | `false` | no |
 | <a name="input_enable_yunikorn"></a> [enable\_yunikorn](#input\_enable\_yunikorn) | Enable Apache YuniKorn Scheduler | `bool` | `false` | no |
 | <a name="input_kms_key_admin_roles"></a> [kms\_key\_admin\_roles](#input\_kms\_key\_admin\_roles) | list of role ARNs to add to the KMS policy | `list(string)` | `[]` | no |
-| <a name="input_name"></a> [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"spark-operator-doeks"` | no |
+| <a name="input_name"></a> [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"spark-eks-s3tables"` | no |
 | <a name="input_private_subnets"></a> [private\_subnets](#input\_private\_subnets) | Private Subnets CIDRs. 254 IPs per Subnet/AZ for Private NAT + NLB + Airflow + EC2 Jumphost etc. | `list(string)` | <pre>[<br>  "10.1.1.0/24",<br>  "10.1.2.0/24"<br>]</pre> | no |
 | <a name="input_public_subnets"></a> [public\_subnets](#input\_public\_subnets) | Public Subnets CIDRs. 62 IPs per Subnet/AZ | `list(string)` | <pre>[<br>  "10.1.0.0/26",<br>  "10.1.0.64/26"<br>]</pre> | no |
 | <a name="input_region"></a> [region](#input\_region) | Region | `string` | `"us-west-2"` | no |

analytics/terraform/spark-k8s-operator/main.tf

@@ -94,3 +94,63 @@ data "aws_iam_policy_document" "spark_operator" {
     ]
   }
 }
+
+#---------------------------------------------------------------------
+# Example IAM policy for s3 Tables access from Spark Jobs.
+# Please modify this policy according to your security requirements.
+#---------------------------------------------------------------------
+data "aws_iam_policy_document" "s3tables_policy" {
+  version = "2012-10-17"
+
+  statement {
+    sid    = "VisualEditor0"
+    effect = "Allow"
+
+    actions = [
+      "s3tables:CreateTableBucket",
+      "s3tables:ListTables",
+      "s3tables:CreateTable",
+      "s3tables:GetNamespace",
+      "s3tables:DeleteTableBucket",
+      "s3tables:CreateNamespace",
+      "s3tables:ListNamespaces",
+      "s3tables:GetTableBucket",
+      "s3tables:DeleteNamespace",
+      "s3tables:GetTableBucketMaintenanceConfiguration",
+      "s3tables:PutTableBucketMaintenanceConfiguration",
+      "s3tables:GetTableBucketPolicy"
+    ]
+
+    resources = ["arn:aws:s3tables:*:${data.aws_caller_identity.current.account_id}:bucket/*"]
+  }
+
+  statement {
+    sid    = "VisualEditor1"
+    effect = "Allow"
+
+    actions = [
+      "s3tables:GetTableMaintenanceJobStatus",
+      "s3tables:GetTablePolicy",
+      "s3tables:GetTable",
+      "s3tables:GetTableMetadataLocation",
+      "s3tables:UpdateTableMetadataLocation",
+      "s3tables:DeleteTable",
+      "s3tables:PutTableData",
+      "s3tables:RenameTable",
+      "s3tables:PutTableMaintenanceConfiguration",
+      "s3tables:GetTableData",
+      "s3tables:GetTableMaintenanceConfiguration"
+    ]
+
+    resources = ["arn:aws:s3tables:*:${data.aws_caller_identity.current.account_id}:bucket/*/table/*"]
+  }
+
+  statement {
+    sid    = "VisualEditor2"
+    effect = "Allow"
+
+    actions = ["s3tables:ListTableBuckets"]
+
+    resources = ["*"]
+  }
+}

analytics/terraform/spark-k8s-operator/spark-team.tf

@@ -52,6 +52,7 @@ module "spark_team_irsa" {
   create_policy  = false
   role_policies = {
     spark_team_policy = aws_iam_policy.spark.arn
+    s3tables_policy   = aws_iam_policy.s3tables.arn
   }
 
   oidc_providers = {
@@ -69,6 +70,12 @@ resource "aws_iam_policy" "spark" {
   policy      = data.aws_iam_policy_document.spark_operator.json
 }
 
+resource "aws_iam_policy" "s3tables" {
+  description = "IAM role policy for S3 Tables Access from Spark Job execution"
+  name_prefix = "${local.name}-s3tables-irsa"
+  policy      = data.aws_iam_policy_document.s3tables_policy.json
+}
+
 resource "kubernetes_cluster_role" "spark_role" {
   metadata {
     name = "spark-cluster-role"