[u] Enable inventory of all S3 buckets (#4715)

DataBiosphere · Jun 5, 2023 · 8ff59c9 · 8ff59c9
1 parent 14db0b5
commit 8ff59c9
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 12 deletions.
diff --git a/UPGRADING.rst b/UPGRADING.rst
@@ -26,8 +26,9 @@ have too many entries in this file.
 Operator
 ~~~~~~~~
 
-Manually deploy the ``gitlab`` component of any main deployment just before
-pushing the merge commit to the GitLab instance in that deployment.
+Manually deploy the ``shared`` and ``gitlab`` components (in that order) of any
+main deployment just before pushing the merge commit to the GitLab instance in
+that deployment.
 
 If deploying the ``gitlab`` component results in an ``OptInRequired`` error,
 login to the AWS Console using credentials for the AWS account that contains the

diff --git a/src/azul/__init__.py b/src/azul/__init__.py
@@ -194,7 +194,9 @@ def qualified_bucket_name(self,
                               region_name: str,
                               bucket_name: str
                               ) -> str:
-        self._validate_term(bucket_name, name='bucket_name')
+        # Allow wildcard for use in ARN patterns
+        if bucket_name != '*':
+            self._validate_term(bucket_name, name='bucket_name')
         return f'edu-ucsc-gi-{account_name}-{bucket_name}.{region_name}'
 
     aws_config_term = 'awsconfig'

diff --git a/src/azul/terraform.py b/src/azul/terraform.py
@@ -333,6 +333,51 @@ def block_public_s3_bucket_access(tf_config: JSON) -> JSON:
     return tf_config
 
 
+def enable_s3_bucket_inventory(tf_config: JSON,
+                               dest_bucket_ref: str = 'data.aws_s3_bucket.logs',
+                               /,
+                               ) -> JSON:
+    key = 'resource'
+    tf_config = copy_json(tf_config, key)
+    tf_config[key]['aws_s3_bucket_inventory'] = {
+        resource_name: {
+            **(
+                {'provider': resource['provider']}
+                if 'provider' in resource else {}
+            ),
+            'bucket': '${aws_s3_bucket.%s.id}' % resource_name,
+            'name': config.qualified_resource_name('inventory'),
+            'included_object_versions': 'All',
+            'destination': {
+                'bucket': {
+                    'format': 'CSV',
+                    'bucket_arn': '${%s.arn}' % dest_bucket_ref,
+                    'prefix': 'inventory'
+                }
+            },
+            'schedule': {
+                'frequency': 'Daily'
+            },
+            'optional_fields': [
+                'Size',
+                'LastModifiedDate',
+                'StorageClass',
+                'ETag',
+                'IsMultipartUploaded',
+                'ReplicationStatus',
+                'EncryptionStatus',
+                'ChecksumAlgorithm',
+                'BucketKeyStatus',
+                'IntelligentTieringAccessTier',
+                'ObjectLockMode',
+                'ObjectLockRetainUntilDate',
+                'ObjectLockLegalHoldStatus'
+            ]
+        } for resource_name, resource in tf_config[key]['aws_s3_bucket'].items()
+    }
+    return tf_config
+
+
 U = TypeVar('U', bound=AnyJSON)
 
 

diff --git a/terraform/browser/browser.tf.json.template.py b/terraform/browser/browser.tf.json.template.py
@@ -31,6 +31,7 @@
 from azul.terraform import (
     block_public_s3_bucket_access,
     emit_tf,
+    enable_s3_bucket_inventory,
 )
 
 buckets = {
@@ -42,7 +43,7 @@
 }
 
 
-def emit(): emit_tf(block_public_s3_bucket_access({
+def emit(): emit_tf(block_public_s3_bucket_access(enable_s3_bucket_inventory({
     'data': {
         'aws_s3_bucket': {
             'logs': {
@@ -334,7 +335,7 @@ def emit(): emit_tf(block_public_s3_bucket_access({
             }
         }
     }
-}))
+})))
 
 
 def bucket_behaviour(origin, *, path_pattern: str = None, **functions: bool) -> JSON:

diff --git a/terraform/s3.tf.json.template.py b/terraform/s3.tf.json.template.py
@@ -7,13 +7,14 @@
 from azul.terraform import (
     block_public_s3_bucket_access,
     emit_tf,
+    enable_s3_bucket_inventory,
 )
 
-emit_tf(block_public_s3_bucket_access({
+tf_config = {
     'data': {
         'aws_s3_bucket': {
             'logs': {
-                'bucket': aws.qualified_bucket_name(config.logs_term),
+                'bucket': aws.logs_bucket,
             }
         },
     },
@@ -52,4 +53,7 @@
             }
         }
     }
-}))
+}
+tf_config = enable_s3_bucket_inventory(tf_config)
+tf_config = block_public_s3_bucket_access(tf_config)
+emit_tf(tf_config)
diff --git a/terraform/shared/shared.tf.json.template.py b/terraform/shared/shared.tf.json.template.py
@@ -13,6 +13,7 @@
 from azul.terraform import (
     block_public_s3_bucket_access,
     emit_tf,
+    enable_s3_bucket_inventory,
     vpc,
 )
 
@@ -105,7 +106,7 @@ def conformance_pack(name: str) -> str:
                                    '$.eventType !="AwsServiceEvent"}')
 ]
 
-emit_tf(block_public_s3_bucket_access({
+tf_config = {
     'data': {
         'aws_iam_role': {
             f'support_{i}': {
@@ -270,10 +271,31 @@ def conformance_pack(name: str) -> str:
                             source_bucket_arn='arn:aws:s3:::*',
                             target_bucket_arn='${aws_s3_bucket.logs.arn}',
                             path_prefix=config.s3_access_log_path_prefix('*', deployment=None)
-                        )
+                        ),
+                        {
+                            'Effect': 'Allow',
+                            'Principal': {
+                                'Service': 's3.amazonaws.com'
+                            },
+                            'Action': [
+                                's3:PutObject'
+                            ],
+                            'Resource': [
+                                'arn:aws:s3:::${aws_s3_bucket.logs.id}/*'
+                            ],
+                            'Condition': {
+                                'ArnLike': {
+                                    'aws:SourceArn': f'arn:aws:s3:::{aws.qualified_bucket_name("*")}'
+                                },
+                                'StringEquals': {
+                                    'aws:SourceAccount': config.aws_account_id,
+                                    's3:x-amz-acl': 'bucket-owner-full-control'
+                                }
+                            }
+                        }
                     ]
                 })
-            }
+            },
         },
         'aws_s3_bucket_lifecycle_configuration': {
             'shared': {
@@ -808,4 +830,7 @@ def conformance_pack(name: str) -> str:
             }
         }
     }
-}))
+}
+tf_config = enable_s3_bucket_inventory(tf_config, 'aws_s3_bucket.logs')
+tf_config = block_public_s3_bucket_access(tf_config)
+emit_tf(tf_config)