updating module S3 Bucket Keys for SSE-KMS

plugins/modules/s3_bucket.py

@@ -84,6 +84,15 @@
     description: KMS master key ID to use for the default encryption. This parameter is allowed if I(encryption) is C(aws:kms). If
                  not specified then it will default to the AWS provided KMS key.
     type: str
+  bucket_key_enabled:
+    description:
+      - Enable S3 Bucket Keys for SSE-KMS on new objects.
+      - See the AWS documentation for more information
+        U(https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucket-key.html).
+      - Bucket Key encryption is only supported if I(encryption=aws:kms).
+    required: false
+    type: bool
+
   public_access:
     description:
       - Configure public access block for S3 bucket.
@@ -214,6 +223,11 @@
     encryption: "aws:kms"
     encryption_key_id: "arn:aws:kms:us-east-1:1234/5678example"
 
+# Create a bucket with aws:kms encryption, Bucket key
+- amazon.aws.s3_bucket:
+    name: mys3bucket
+    bucket_key_enabled: true
+
 # Create a bucket with aws:kms encryption, default key
 - amazon.aws.s3_bucket:
     name: mys3bucket
@@ -284,6 +298,7 @@
     type: dict
     returned: I(state=present)
     sample: {
+
         "Statement": [
             {
                 "Action": "s3:GetObject",
@@ -359,6 +374,7 @@ def create_or_update_bucket(s3_client, module, location):
     versioning = module.params.get("versioning")
     encryption = module.params.get("encryption")
     encryption_key_id = module.params.get("encryption_key_id")
+    bucket_key_enabled = module.params.get("bucket_key_enabled")
     public_access = module.params.get("public_access")
     delete_public_access = module.params.get("delete_public_access")
     delete_object_ownership = module.params.get("delete_object_ownership")
@@ -535,8 +551,17 @@ def create_or_update_bucket(s3_client, module, location):
                     current_encryption = put_bucket_encryption_with_retry(module, s3_client, name, expected_encryption)
                     changed = True
 
+        if bucket_key_enabled is not None:
+            current_encryption_algorithm = current_encryption.get('SSEAlgorithm') if current_encryption else None
+            if current_encryption_algorithm == 'aws:kms':
+                if get_bucket_key(s3_client, name) != bucket_key_enabled:
+                    if bucket_key_enabled:
+                        expected_encryption = True
+                    else:
+                        expected_encryption = False
+                    current_encryption = put_bucket_key_with_retry(module, s3_client, name, expected_encryption)
+                    changed = True
         result['encryption'] = current_encryption
-
     # Public access clock configuration
     current_public_access = {}
 
@@ -701,6 +726,17 @@ def get_bucket_encryption(s3_client, bucket_name):
         return None
 
 
+@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted'])
+def get_bucket_key(s3_client, bucket_name):
+    try:
+        result = s3_client.get_bucket_encryption(Bucket=bucket_name)
+        return result.get('ServerSideEncryptionConfiguration', {}).get('Rules', [])[0].get('BucketKeyEnabled')
+    except is_boto3_error_code('ServerSideEncryptionConfigurationNotFoundError'):
+        return None
+    except (IndexError, KeyError):
+        return None
+
+
 def put_bucket_encryption_with_retry(module, s3_client, name, expected_encryption):
     max_retries = 3
     for retries in range(1, max_retries + 1):
@@ -726,6 +762,38 @@ def put_bucket_encryption(s3_client, bucket_name, encryption):
     s3_client.put_bucket_encryption(Bucket=bucket_name, ServerSideEncryptionConfiguration=server_side_encryption_configuration)
 
 
+def put_bucket_key_with_retry(module, s3_client, name, expected_encryption):
+    max_retries = 3
+    for retries in range(1, max_retries + 1):
+        try:
+            put_bucket_key(s3_client, name, expected_encryption)
+        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:  # pylint: disable=duplicate-except
+            module.fail_json_aws(e, msg="Failed to set bucket Key")
+        result = s3_client.get_bucket_encryption(Bucket=name)
+        current_encryption = wait_bucket_key_is_applied(module, s3_client, name, expected_encryption,
+                                                        should_fail=(retries == max_retries), retries=5)
+        if current_encryption == expected_encryption:
+            return result
+
+    # We shouldn't get here, the only time this should happen is if
+    # current_encryption != expected_encryption and retries == max_retries
+    # Which should use module.fail_json and fail out first.
+    module.fail_json(msg='Failed to set bucket key',
+                     current=current_encryption, expected=expected_encryption, retries=retries)
+
+
+@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted'])
+def put_bucket_key(s3_client, bucket_name, encryption):
+    # server_side_encryption_configuration ={'Rules': [{'BucketKeyEnabled': encryption}]}
+    encryption_status = s3_client.get_bucket_encryption(Bucket=bucket_name)
+    encryption_status['ServerSideEncryptionConfiguration']['Rules'][0]['BucketKeyEnabled'] = encryption
+    s3_client.put_bucket_encryption(
+        Bucket=bucket_name,
+        ServerSideEncryptionConfiguration=encryption_status[
+            'ServerSideEncryptionConfiguration']
+    )
+
+
 @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted'])
 def delete_bucket_tagging(s3_client, bucket_name):
     s3_client.delete_bucket_tagging(Bucket=bucket_name)
@@ -835,6 +903,23 @@ def wait_encryption_is_applied(module, s3_client, bucket_name, expected_encrypti
     return encryption
 
 
+def wait_bucket_key_is_applied(module, s3_client, bucket_name, expected_encryption, should_fail=True, retries=12):
+    for dummy in range(0, retries):
+        try:
+            encryption = get_bucket_key(s3_client, bucket_name)
+        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+            module.fail_json_aws(e, msg="Failed to get updated encryption for bucket")
+        if encryption != expected_encryption:
+            time.sleep(5)
+        else:
+            return encryption
+
+    if should_fail:
+        module.fail_json(msg="Bucket Key failed to apply in the expected time",
+                         requested_encryption=expected_encryption, live_encryption=encryption)
+    return encryption
+
+
 def wait_versioning_is_applied(module, s3_client, bucket_name, required_versioning):
     for dummy in range(0, 24):
         try:
@@ -1009,6 +1094,7 @@ def main():
         ceph=dict(default=False, type='bool'),
         encryption=dict(choices=['none', 'AES256', 'aws:kms']),
         encryption_key_id=dict(),
+        bucket_key_enabled=dict(type='bool'),
         public_access=dict(type='dict', options=dict(
             block_public_acls=dict(type='bool', default=False),
             ignore_public_acls=dict(type='bool', default=False),
@@ -1070,6 +1156,7 @@ def main():
     state = module.params.get("state")
     encryption = module.params.get("encryption")
     encryption_key_id = module.params.get("encryption_key_id")
+    bucket_key_enabled = module.params.get("bucket_key_enabled")
     delete_object_ownership = module.params.get('delete_object_ownership')
     object_ownership = module.params.get('object_ownership')
 

tests/integration/targets/s3_bucket/inventory

@@ -6,6 +6,7 @@ complex
 dotted
 tags
 encryption_kms
+encryption_bucket_key
 encryption_sse
 public_access
 acl

tests/integration/targets/s3_bucket/roles/s3_bucket/tasks/encryption_bucket_key.yml

@@ -0,0 +1,100 @@
+---
+- module_defaults:
+    group/aws:
+      aws_access_key: "{{ aws_access_key }}"
+      aws_secret_key: "{{ aws_secret_key }}"
+      security_token: "{{ security_token | default(omit) }}"
+      region: "{{ aws_region }}"
+  block:
+    - name: Set facts for encryption_bucket_key test
+      set_fact:
+        local_bucket_name: "{{ bucket_name | hash('md5') }}-bucket-key"
+    # ============================================================
+
+    - name: "Create a simple bucket"
+      s3_bucket:
+        name: "{{ local_bucket_name }}"
+        state: present
+      register: output
+
+    - name: "Enable aws:kms encryption with KMS master key"
+      s3_bucket:
+        name: "{{ local_bucket_name }}"
+        state: present
+        encryption: "aws:kms"
+      register: output
+
+    - name: "Enable bucket key for bucket with aws:kms encryption"
+      s3_bucket:
+        name: "{{ local_bucket_name }}"
+        state: present
+        encryption: "aws:kms"
+        bucket_key_enabled: true
+      register: output
+
+    - name: Assert for 'Enable bucket key for bucket with aws:kms encryption'
+      assert:
+        that:
+          - output.changed
+          - output.encryption
+
+    - name: "Re-enable bucket key for bucket with aws:kms encryption (idempotent)"
+      s3_bucket:
+        name: "{{ local_bucket_name }}"
+        encryption: "aws:kms"
+        bucket_key_enabled: true
+      register: output
+
+    - name: Assert for 'Re-enable bucket key for bucket with aws:kms encryption (idempotent)''
+      assert:
+        that:
+          - not output.changed
+          - output.encryption
+
+    # ============================================================
+
+    - name: Disable encryption from bucket
+      s3_bucket:
+        name: "{{ local_bucket_name }}"
+        encryption: none
+        bucket_key_enabled: false
+      register: output
+
+    - name: Assert for 'Disable encryption from bucket'
+      assert:
+        that:
+          - output.changed
+          - not output.encryption
+
+    - name: Disable encryption from bucket (idempotent)
+      s3_bucket:
+        name: "{{ local_bucket_name }}"
+        bucket_key_enabled: true
+      register: output
+
+    - name: Assert for 'Disable encryption from bucket (idempotent)'
+      assert:
+        that:
+          - output is not changed
+          - not output.encryption
+
+    # ============================================================
+
+    - name: Delete encryption test s3 bucket
+      s3_bucket:
+        name: "{{ local_bucket_name }}"
+        state: absent
+      register: output
+
+    - name: Assert for 'Delete encryption test s3 bucket'
+      assert:
+        that:
+          - output.changed
+
+  # ============================================================
+  always:
+    - name: Ensure all buckets are deleted
+      s3_bucket:
+        name: "{{ local_bucket_name }}"
+        state: absent
+      failed_when: false