aws_ec2 inventory: include_filters and exclude_filters (#328)

aws_ec2 inventory: add includes_filters and excludes_filters

Reviewed-by: https://github.com/apps/ansible-zuul

changelogs/fragments/328_aws_ec2_inventory_includes_entries_matching.yaml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+- "aws_ec2 inventory - expose to new keys called ``include_filters`` and ``exclude_filters`` to give the user the ability to compose an inventory with multiple queries (https://github.com/ansible-collections/amazon.aws/pull/328)."

plugins/inventory/aws_ec2.py

@@ -51,6 +51,23 @@
               - Available filters are listed here U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options).
           type: dict
           default: {}
+        include_filters:
+          description:
+              - A list of filters. Any instances matching at least one of the filters are included in the result.
+              - Available filters are listed here U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options).
+              - Every entry in this list triggers a search query. As such, from a performance point of view, it's better to
+                keep the list as short as possible.
+          type: list
+          default: []
+        exclude_filters:
+          description:
+              - A list of filters. Any instances matching one of the filters are excluded from the result.
+              - The filters from C(exclude_filters) take priority over the C(include_filters) and C(filters) keys
+              - Available filters are listed here U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options).
+              - Every entry in this list triggers a search query. As such, from a performance point of view, it's better to
+                keep the list as short as possible.
+          type: list
+          default: []
         include_extra_api_calls:
           description:
               - Add two additional API calls for every instance to include 'persistent' and 'events' host variables.
@@ -157,6 +174,20 @@
   # Use the private IP address to connect to the host
   # (note: this does not modify inventory_hostname, which is set via I(hostnames))
   ansible_host: private_ip_address
+
+# Example using include_filters and exclude_filters to compose the inventory.
+plugin: aws_ec2
+regions:
+  - us-east-1
+  - us-west-1
+include_filters:
+- tag:Name:
+  - 'my_second_tag'
+- tag:Name:
+  - 'my_third_tag'
+exclude_filters:
+- tag:Name:
+  - 'my_first_tag'
 '''
 
 import re
@@ -462,7 +493,7 @@ def _get_instances_by_region(self, regions, filters, strict_permissions):
 
             all_instances.extend(instances)
 
-        return sorted(all_instances, key=lambda x: x['InstanceId'])
+        return all_instances
 
     def _get_reservation_details(self, reservation):
         return {
@@ -543,14 +574,34 @@ def _get_hostname(self, instance, hostnames):
             else:
                 return to_text(hostname)
 
-    def _query(self, regions, filters, strict_permissions):
+    def _query(self, regions, include_filters, exclude_filters, strict_permissions):
         '''
             :param regions: a list of regions to query
-            :param filters: a list of boto3 filter dictionaries
-            :param hostnames: a list of hostname destination variables in order of preference
+            :param include_filters: a list of boto3 filter dictionaries
+            :param exclude_filters: a list of boto3 filter dictionaries
             :param strict_permissions: a boolean determining whether to fail or ignore 403 error codes
+
         '''
-        return {'aws_ec2': self._get_instances_by_region(regions, filters, strict_permissions)}
+        instances = []
+        ids_to_ignore = []
+        for filter in exclude_filters:
+            for i in self._get_instances_by_region(
+                    regions,
+                    ansible_dict_to_boto3_filter_list(filter),
+                    strict_permissions):
+                ids_to_ignore.append(i['InstanceId'])
+        for filter in include_filters:
+            for i in self._get_instances_by_region(
+                    regions,
+                    ansible_dict_to_boto3_filter_list(filter),
+                    strict_permissions):
+                if i['InstanceId'] not in ids_to_ignore:
+                    instances.append(i)
+                    ids_to_ignore.append(i['InstanceId'])
+
+        instances = sorted(instances, key=lambda x: x['InstanceId'])
+
+        return {'aws_ec2': instances}
 
     def _populate(self, groups, hostnames):
         for group in groups:
@@ -659,7 +710,8 @@ def parse(self, inventory, loader, path, cache=True):
 
         # get user specifications
         regions = self.get_option('regions')
-        filters = ansible_dict_to_boto3_filter_list(self.get_option('filters'))
+        include_filters = [self.get_option('filters')] + self.get_option('include_filters')
+        exclude_filters = self.get_option('exclude_filters')
         hostnames = self.get_option('hostnames')
         strict_permissions = self.get_option('strict_permissions')
 
@@ -679,7 +731,7 @@ def parse(self, inventory, loader, path, cache=True):
                 cache_needs_update = True
 
         if not cache or cache_needs_update:
-            results = self._query(regions, filters, strict_permissions)
+            results = self._query(regions, include_filters, exclude_filters, strict_permissions)
 
         self._populate(results, hostnames)
 

tests/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory_with_include_or_exclude_filters.yml

@@ -0,0 +1,118 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  gather_facts: no
+  environment: "{{ ansible_test.environment }}"
+  tasks:
+
+    - module_defaults:
+        group/aws:
+          aws_access_key: '{{ aws_access_key }}'
+          aws_secret_key: '{{ aws_secret_key }}'
+          security_token: '{{ security_token | default(omit) }}'
+          region: '{{ aws_region }}'
+      block:
+
+        # Create VPC, subnet, security group, and find image_id to create instance
+
+        - include_tasks: setup.yml
+
+        # Create new host, refresh inventory
+        - name: create a new host (1/3)
+          ec2:
+            image: '{{ image_id }}'
+            exact_count: 1
+            count_tag:
+              Name: '{{ resource_prefix }}_1'
+            instance_tags:
+              Name: '{{ resource_prefix }}_1'
+              tag_instance1: foo
+            instance_type: t2.micro
+            wait: yes
+            group_id: '{{ sg_id }}'
+            vpc_subnet_id: '{{ subnet_id }}'
+          register: setup_instance_1
+
+        - name: create a new host (2/3)
+          ec2:
+            image: '{{ image_id }}'
+            exact_count: 1
+            count_tag:
+              Name: '{{ resource_prefix }}_2'
+            instance_tags:
+              Name: '{{ resource_prefix }}_2'
+              tag_instance2: bar
+            instance_type: t2.micro
+            wait: yes
+            group_id: '{{ sg_id }}'
+            vpc_subnet_id: '{{ subnet_id }}'
+          register: setup_instance_2
+
+        - name: create a new host (3/3)
+          ec2:
+            image: '{{ image_id }}'
+            exact_count: 1
+            count_tag:
+              Name: '{{ resource_prefix }}_3'
+            instance_tags:
+              Name: '{{ resource_prefix }}_3'
+              tag_instance2: bar
+            instance_type: t2.micro
+            wait: yes
+            group_id: '{{ sg_id }}'
+            vpc_subnet_id: '{{ subnet_id }}'
+          register: setup_instance_3
+
+        - meta: refresh_inventory
+
+        - name: assert the keyed groups and groups from constructed config were added to inventory and composite var added to hostvars
+          assert:
+            that:
+              # There are 9 groups: all, ungrouped, aws_ec2, sg keyed group, 3 tag keyed group (one per tag), arch keyed group, constructed group
+              - "groups['all'] | length == 2"
+              - "'{{ resource_prefix }}_1' in groups['all']"
+              - "'{{ resource_prefix }}_2' in groups['all']"
+              - "not ('{{ resource_prefix }}_3' in groups['all'])"
+
+      always:
+
+        - name: remove setup ec2 instance (1/3)
+          ec2:
+            instance_type: t2.micro
+            instance_ids: '{{ setup_instance_1.instance_ids }}'
+            state: absent
+            wait: yes
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+            group_id: "{{ sg_id }}"
+            vpc_subnet_id: "{{ subnet_id }}"
+          ignore_errors: yes
+          when: setup_instance_1 is defined
+
+        - name: remove setup ec2 instance (2/3)
+          ec2:
+            instance_type: t2.micro
+            instance_ids: '{{ setup_instance_2.instance_ids }}'
+            state: absent
+            wait: yes
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+            group_id: "{{ sg_id }}"
+            vpc_subnet_id: "{{ subnet_id }}"
+          ignore_errors: yes
+          when: setup_instance_2 is defined
+
+        - name: remove setup ec2 instance (3/3)
+          ec2:
+            instance_type: t2.micro
+            instance_ids: '{{ setup_instance_3.instance_ids }}'
+            state: absent
+            wait: yes
+            instance_tags:
+              Name: '{{ resource_prefix }}'
+            group_id: "{{ sg_id }}"
+            vpc_subnet_id: "{{ subnet_id }}"
+          ignore_errors: yes
+          when: setup_instance_3 is defined
+
+        - include_tasks: tear_down.yml

tests/integration/targets/inventory_aws_ec2/runme.sh

@@ -38,6 +38,9 @@ ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_w
 ansible-playbook playbooks/test_populating_inventory_with_constructed.yml "$@"
 ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_concatenation.yml.j2'" "$@"
 ansible-playbook playbooks/test_populating_inventory_with_concatenation.yml "$@"
+# generate inventory config with includes_entries_matching and prepare the tests
+ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_include_or_exclude_filters.yml.j2'" "$@"
+ansible-playbook playbooks/test_populating_inventory_with_include_or_exclude_filters.yml "$@"
 
 # generate inventory config with caching and test using it
 ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_use_contrib_script_keys.yml.j2'" "$@"

tests/integration/targets/inventory_aws_ec2/templates/inventory_with_include_or_exclude_filters.yml.j2

@@ -0,0 +1,23 @@
+plugin: amazon.aws.aws_ec2
+aws_access_key_id: '{{ aws_access_key }}'
+aws_secret_access_key: '{{ aws_secret_key }}'
+{% if security_token | default(false) %}
+aws_security_token: '{{ security_token }}'
+{% endif %}
+regions:
+- '{{ aws_region }}'
+filters:
+  tag:Name:
+  - '{{ resource_prefix }}_1'
+  - '{{ resource_prefix }}_3'
+include_filters:
+- tag:Name:
+  - '{{ resource_prefix }}_2'
+  - '{{ resource_prefix }}_4'
+exclude_filters:
+- tag:Name:
+  - '{{ resource_prefix }}_3'
+  - '{{ resource_prefix }}_4'
+hostnames:
+- tag:Name
+- dns-name