Add support for managing S3 logging

plugins/modules/elb_classic_lb.py

@@ -155,8 +155,39 @@
         type: int
   access_logs:
     description:
-      - An associative array of access logs configuration settings (see examples).
+      - A dictionary of access logs configuration settings (see examples).
     type: dict
+    suboptions:
+      enabled:
+        description:
+        - When set to C(True) will configure delivery of access logs to an S3
+          bucket.
+        - When set to C(False) will disable delivery of access logs.
+        required: false
+        type: bool
+        default: true
+      s3_location:
+        description:
+        - The S3 bucket to deliver access logs to.
+        - See U(https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html)
+          for more information about the necessary S3 bucket policies.
+        - Required when I(enabled=True).
+        required: false
+        type: str
+      s3_prefix:
+        description:
+        - Where in the S3 bucket to deliver the logs.
+        - If the prefix is not provided or set to C(""), the log is placed at the root level of the bucket.
+        required: false
+        type: str
+        default: ""
+      interval:
+        description:
+        - The interval for publishing the access logs to S3.
+        required: false
+        type: int
+        default: 60
+        choices: [ 5, 60 ]
   subnets:
     description:
       - A list of VPC subnets to use when creating the ELB.
@@ -1242,6 +1273,22 @@ def _set_elb_attributes(self):
             if not curr_attr == attr:
                 attributes['ConnectionDraining'] = attr
 
+        if self.access_logs is not None:
+            curr_attr = dict(self.elb_attributes.get('AccessLog', {}))
+            # For disabling we only need to compare and pass 'Enabled'
+            if not self.access_logs.get('enabled'):
+                curr_attr = dict(Enabled=curr_attr.get('Enabled', False))
+                attr = dict(Enabled=self.access_logs.get('enabled'))
+            else:
+                attr = dict(
+                    Enabled=True,
+                    S3BucketName=self.access_logs['s3_location'],
+                    S3BucketPrefix=self.access_logs.get('s3_prefix', ''),
+                    EmitInterval=self.access_logs.get('interval', 60),
+                )
+            if not curr_attr == attr:
+                attributes['AccessLog'] = attr
+
         if not attributes:
             return False
 
@@ -1258,32 +1305,6 @@ def _set_elb_attributes(self):
         except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
             self.module.fail_json_aws(e, msg="Failed to apply load balancer attrbutes")
 
-#    def _set_access_log(self):
-#        attributes = self.elb.get_attributes()
-#        if self.access_logs:
-#            if 's3_location' not in self.access_logs:
-#                self.module.fail_json(msg='s3_location information required')
-#
-#            access_logs_config = {
-#                "enabled": True,
-#                "s3_bucket_name": self.access_logs['s3_location'],
-#                "s3_bucket_prefix": self.access_logs.get('s3_prefix', ''),
-#                "emit_interval": self.access_logs.get('interval', 60),
-#            }
-#
-#            update_access_logs_config = False
-#            for attr, desired_value in access_logs_config.items():
-#                if getattr(attributes.access_log, attr) != desired_value:
-#                    setattr(attributes.access_log, attr, desired_value)
-#                    update_access_logs_config = True
-#            if update_access_logs_config:
-#                self.elb_conn.modify_lb_attribute(self.name, 'AccessLog', attributes.access_log)
-#                self.changed = True
-#        elif attributes.access_log.enabled:
-#            attributes.access_log.enabled = False
-#            self.changed = True
-#            self.elb_conn.modify_lb_attribute(self.name, 'AccessLog', attributes.access_log)
-
     def _proxy_policy_name(self):
         return 'ProxyProtocol-policy'
 
@@ -1760,6 +1781,13 @@ def _describe_loadbalancer(self, lb_name):
 
 def main():
 
+    access_log_spec = dict(
+        enabled=dict(required=False, type='bool', default=True),
+        s3_location=dict(required=False, type='str'),
+        s3_prefix=dict(required=False, type='str', default=""),
+        interval=dict(required=False, type='int', default=60, choices=[5, 60]),
+    )
+
     stickiness_spec = dict(
         type=dict(required=False, type='str', choices=['application', 'loadbalancer']),
         enabled=dict(required=False, type='bool', default=True),
@@ -1808,7 +1836,7 @@ def main():
         idle_timeout=dict(type='int'),
         cross_az_load_balancing=dict(type='bool'),
         stickiness=dict(type='dict', options=stickiness_spec),
-        access_logs=dict(type='dict'),
+        access_logs=dict(type='dict', options=access_log_spec),
         wait=dict(default=False, type='bool'),
         wait_timeout=dict(default=180, type='int'),
         tags=dict(type='dict'),

tests/integration/targets/elb_classic_lb/defaults/main.yml

@@ -140,3 +140,24 @@ lb_stickiness:
 updated_lb_stickiness:
     type: loadbalancer
     expiration: 600
+
+# Amazon's SDKs don't provide the list of account ID's.  Amazon only provide a
+# web page.  If you want to run the tests outside the US regions you'll need to
+# update this.
+# https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html
+access_log_account_id_map:
+    us-east-1: '127311923021'
+    us-east-2: '033677994240'
+    us-west-1: '027434742980'
+    us-west-2: '797873946194'
+    us-gov-west-1: '048591011584'
+    us-gov-east-1: '190560391635'
+
+access_log_account_id: '{{ access_log_account_id_map[aws_region] }}'
+
+s3_logging_bucket_a: '{{ resource_prefix }}-a'
+s3_logging_bucket_b: '{{ resource_prefix }}-b'
+default_logging_prefix: 'logs'
+updated_logging_prefix: 'mylogs'
+default_logging_interval: 5
+updated_logging_interval: 60

tests/integration/targets/elb_classic_lb/tasks/cleanup_s3.yml

@@ -0,0 +1,32 @@
+---
+- name: Create empty temporary directory
+  tempfile:
+    state: directory
+  register: tmpdir
+  ignore_errors: true
+
+- name: Empty S3 buckets before deletion
+  s3_sync:
+    bucket: '{{ item }}'
+    delete: true
+    file_root: '{{ tmpdir.path }}'
+  ignore_errors: true
+  loop:
+    - '{{ s3_logging_bucket_a }}'
+    - '{{ s3_logging_bucket_b }}'
+
+- name: Delete S3 bucket for access logs
+  s3_bucket:
+    name: '{{ item }}'
+    state: absent
+  register: logging_bucket
+  ignore_errors: true
+  loop:
+    - '{{ s3_logging_bucket_a }}'
+    - '{{ s3_logging_bucket_b }}'
+
+- name: Remove temporary directory
+  file:
+    state: absent
+    path: "{{ tmpdir.path }}"
+  ignore_errors: yes

tests/integration/targets/elb_classic_lb/tasks/main.yml

@@ -29,6 +29,7 @@
     - include_tasks: describe_region.yml
     - include_tasks: setup_vpc.yml
     - include_tasks: setup_instances.yml
+    - include_tasks: setup_s3.yml
 
     - include_tasks: basic_public.yml
     - include_tasks: basic_internal.yml
@@ -48,5 +49,6 @@
       register: result
       ignore_errors: true
 
+    - include_tasks: cleanup_s3.yml
     - include_tasks: cleanup_instances.yml
     - include_tasks: cleanup_vpc.yml

tests/integration/targets/elb_classic_lb/tasks/setup_s3.yml

@@ -0,0 +1,26 @@
+---
+- name: Create S3 bucket for access logs
+  vars:
+    s3_logging_bucket: '{{ s3_logging_bucket_a }}'
+  s3_bucket:
+    name: '{{ s3_logging_bucket_a }}'
+    state: present
+    policy: "{{ lookup('template','s3_policy.j2') }}"
+  register: logging_bucket
+
+- assert:
+    that:
+      - logging_bucket is changed
+
+- name: Create S3 bucket for access logs
+  vars:
+    s3_logging_bucket: '{{ s3_logging_bucket_b }}'
+  s3_bucket:
+    name: '{{ s3_logging_bucket_b }}'
+    state: present
+    policy: "{{ lookup('template','s3_policy.j2') }}"
+  register: logging_bucket
+
+- assert:
+    that:
+      - logging_bucket is changed

tests/integration/targets/elb_classic_lb/tasks/simple_changes.yml

@@ -16,6 +16,11 @@
         cross_az_load_balancing: True
         idle_timeout: '{{ default_idle_timeout }}'
         connection_draining_timeout: '{{ default_drain_timeout }}'
+        access_logs:
+          interval: '{{ default_logging_interval }}'
+          s3_location: '{{ s3_logging_bucket_a }}'
+          s3_prefix: '{{ default_logging_prefix }}'
+          enabled: true
       register: result
 
     - name: Verify that simple parameters were set
@@ -42,9 +47,14 @@
           - result.elb.idle_timeout == default_idle_timeout
           - result.elb.connection_draining_timeout == default_drain_timeout
           - result.elb.proxy_policy == None
+          - result.load_balancer.load_balancer_attributes.access_log.emit_interval == default_logging_interval
+          - result.load_balancer.load_balancer_attributes.access_log.s3_bucket_name == s3_logging_bucket_a
+          - result.load_balancer.load_balancer_attributes.access_log.s3_bucket_prefix == default_logging_prefix
+          - result.load_balancer.load_balancer_attributes.access_log.enabled == True
 
     ## AZ / Subnet changes are tested in wth the public/internal tests
     ## because they depend on the scheme of the LB
+
     - include_tasks: 'simple_securitygroups.yml'
     - include_tasks: 'simple_listeners.yml'
     - include_tasks: 'simple_healthcheck.yml'
@@ -55,6 +65,7 @@
     - include_tasks: 'simple_proxy_policy.yml'
     - include_tasks: 'simple_stickiness.yml'
     - include_tasks: 'simple_instances.yml'
+    - include_tasks: 'simple_logging.yml'
 
   always: