diff --git a/changelogs/fragments/328_aws_ec2_inventory_includes_entries_matching.yaml b/changelogs/fragments/328_aws_ec2_inventory_includes_entries_matching.yaml new file mode 100644 index 00000000000..a2c0c7cdac6 --- /dev/null +++ b/changelogs/fragments/328_aws_ec2_inventory_includes_entries_matching.yaml @@ -0,0 +1,3 @@ +--- +minor_changes: +- "aws_ec2 inventory - expose to new keys called ``include_filters`` and ``exclude_filters`` to give the user the ability to compose an inventory with multiple queries (https://github.com/ansible-collections/amazon.aws/pull/328)." diff --git a/plugins/inventory/aws_ec2.py b/plugins/inventory/aws_ec2.py index 26a9e8bc107..71a7ec30f23 100644 --- a/plugins/inventory/aws_ec2.py +++ b/plugins/inventory/aws_ec2.py @@ -51,6 +51,23 @@ - Available filters are listed here U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options). type: dict default: {} + include_filters: + description: + - A list of filters. Any instances matching at least one of the filters are included in the result. + - Available filters are listed here U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options). + - Every entry in this list triggers a search query. As such, from a performance point of view, it's better to + keep the list as short as possible. + type: list + default: [] + exclude_filters: + description: + - A list of filters. Any instances matching one of the filters are excluded from the result. + - The filters from C(exclude_filters) take priority over the C(include_filters) and C(filters) keys + - Available filters are listed here U(http://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options). + - Every entry in this list triggers a search query. As such, from a performance point of view, it's better to + keep the list as short as possible. + type: list + default: [] include_extra_api_calls: description: - Add two additional API calls for every instance to include 'persistent' and 'events' host variables. @@ -157,6 +174,20 @@ # Use the private IP address to connect to the host # (note: this does not modify inventory_hostname, which is set via I(hostnames)) ansible_host: private_ip_address + +# Example using include_filters and exclude_filters to compose the inventory. +plugin: aws_ec2 +regions: + - us-east-1 + - us-west-1 +include_filters: +- tag:Name: + - 'my_second_tag' +- tag:Name: + - 'my_third_tag' +exclude_filters: +- tag:Name: + - 'my_first_tag' ''' import re @@ -462,7 +493,7 @@ def _get_instances_by_region(self, regions, filters, strict_permissions): all_instances.extend(instances) - return sorted(all_instances, key=lambda x: x['InstanceId']) + return all_instances def _get_reservation_details(self, reservation): return { @@ -543,14 +574,34 @@ def _get_hostname(self, instance, hostnames): else: return to_text(hostname) - def _query(self, regions, filters, strict_permissions): + def _query(self, regions, include_filters, exclude_filters, strict_permissions): ''' :param regions: a list of regions to query - :param filters: a list of boto3 filter dictionaries - :param hostnames: a list of hostname destination variables in order of preference + :param include_filters: a list of boto3 filter dictionaries + :param exclude_filters: a list of boto3 filter dictionaries :param strict_permissions: a boolean determining whether to fail or ignore 403 error codes + ''' - return {'aws_ec2': self._get_instances_by_region(regions, filters, strict_permissions)} + instances = [] + ids_to_ignore = [] + for filter in exclude_filters: + for i in self._get_instances_by_region( + regions, + ansible_dict_to_boto3_filter_list(filter), + strict_permissions): + ids_to_ignore.append(i['InstanceId']) + for filter in include_filters: + for i in self._get_instances_by_region( + regions, + ansible_dict_to_boto3_filter_list(filter), + strict_permissions): + if i['InstanceId'] not in ids_to_ignore: + instances.append(i) + ids_to_ignore.append(i['InstanceId']) + + instances = sorted(instances, key=lambda x: x['InstanceId']) + + return {'aws_ec2': instances} def _populate(self, groups, hostnames): for group in groups: @@ -659,7 +710,8 @@ def parse(self, inventory, loader, path, cache=True): # get user specifications regions = self.get_option('regions') - filters = ansible_dict_to_boto3_filter_list(self.get_option('filters')) + include_filters = [self.get_option('filters')] + self.get_option('include_filters') + exclude_filters = self.get_option('exclude_filters') hostnames = self.get_option('hostnames') strict_permissions = self.get_option('strict_permissions') @@ -679,7 +731,7 @@ def parse(self, inventory, loader, path, cache=True): cache_needs_update = True if not cache or cache_needs_update: - results = self._query(regions, filters, strict_permissions) + results = self._query(regions, include_filters, exclude_filters, strict_permissions) self._populate(results, hostnames) diff --git a/tests/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory_with_include_or_exclude_filters.yml b/tests/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory_with_include_or_exclude_filters.yml new file mode 100644 index 00000000000..9c1bac81d0e --- /dev/null +++ b/tests/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory_with_include_or_exclude_filters.yml @@ -0,0 +1,118 @@ +--- +- hosts: 127.0.0.1 + connection: local + gather_facts: no + environment: "{{ ansible_test.environment }}" + tasks: + + - module_defaults: + group/aws: + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token | default(omit) }}' + region: '{{ aws_region }}' + block: + + # Create VPC, subnet, security group, and find image_id to create instance + + - include_tasks: setup.yml + + # Create new host, refresh inventory + - name: create a new host (1/3) + ec2: + image: '{{ image_id }}' + exact_count: 1 + count_tag: + Name: '{{ resource_prefix }}_1' + instance_tags: + Name: '{{ resource_prefix }}_1' + tag_instance1: foo + instance_type: t2.micro + wait: yes + group_id: '{{ sg_id }}' + vpc_subnet_id: '{{ subnet_id }}' + register: setup_instance_1 + + - name: create a new host (2/3) + ec2: + image: '{{ image_id }}' + exact_count: 1 + count_tag: + Name: '{{ resource_prefix }}_2' + instance_tags: + Name: '{{ resource_prefix }}_2' + tag_instance2: bar + instance_type: t2.micro + wait: yes + group_id: '{{ sg_id }}' + vpc_subnet_id: '{{ subnet_id }}' + register: setup_instance_2 + + - name: create a new host (3/3) + ec2: + image: '{{ image_id }}' + exact_count: 1 + count_tag: + Name: '{{ resource_prefix }}_3' + instance_tags: + Name: '{{ resource_prefix }}_3' + tag_instance2: bar + instance_type: t2.micro + wait: yes + group_id: '{{ sg_id }}' + vpc_subnet_id: '{{ subnet_id }}' + register: setup_instance_3 + + - meta: refresh_inventory + + - name: assert the keyed groups and groups from constructed config were added to inventory and composite var added to hostvars + assert: + that: + # There are 9 groups: all, ungrouped, aws_ec2, sg keyed group, 3 tag keyed group (one per tag), arch keyed group, constructed group + - "groups['all'] | length == 2" + - "'{{ resource_prefix }}_1' in groups['all']" + - "'{{ resource_prefix }}_2' in groups['all']" + - "not ('{{ resource_prefix }}_3' in groups['all'])" + + always: + + - name: remove setup ec2 instance (1/3) + ec2: + instance_type: t2.micro + instance_ids: '{{ setup_instance_1.instance_ids }}' + state: absent + wait: yes + instance_tags: + Name: '{{ resource_prefix }}' + group_id: "{{ sg_id }}" + vpc_subnet_id: "{{ subnet_id }}" + ignore_errors: yes + when: setup_instance_1 is defined + + - name: remove setup ec2 instance (2/3) + ec2: + instance_type: t2.micro + instance_ids: '{{ setup_instance_2.instance_ids }}' + state: absent + wait: yes + instance_tags: + Name: '{{ resource_prefix }}' + group_id: "{{ sg_id }}" + vpc_subnet_id: "{{ subnet_id }}" + ignore_errors: yes + when: setup_instance_2 is defined + + - name: remove setup ec2 instance (3/3) + ec2: + instance_type: t2.micro + instance_ids: '{{ setup_instance_3.instance_ids }}' + state: absent + wait: yes + instance_tags: + Name: '{{ resource_prefix }}' + group_id: "{{ sg_id }}" + vpc_subnet_id: "{{ subnet_id }}" + ignore_errors: yes + when: setup_instance_3 is defined + + - include_tasks: tear_down.yml diff --git a/tests/integration/targets/inventory_aws_ec2/runme.sh b/tests/integration/targets/inventory_aws_ec2/runme.sh index 38e48fa090f..0d40786dd19 100755 --- a/tests/integration/targets/inventory_aws_ec2/runme.sh +++ b/tests/integration/targets/inventory_aws_ec2/runme.sh @@ -38,6 +38,9 @@ ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_w ansible-playbook playbooks/test_populating_inventory_with_constructed.yml "$@" ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_concatenation.yml.j2'" "$@" ansible-playbook playbooks/test_populating_inventory_with_concatenation.yml "$@" +# generate inventory config with includes_entries_matching and prepare the tests +ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_include_or_exclude_filters.yml.j2'" "$@" +ansible-playbook playbooks/test_populating_inventory_with_include_or_exclude_filters.yml "$@" # generate inventory config with caching and test using it ansible-playbook playbooks/create_inventory_config.yml -e "template='inventory_with_use_contrib_script_keys.yml.j2'" "$@" diff --git a/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_include_or_exclude_filters.yml.j2 b/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_include_or_exclude_filters.yml.j2 new file mode 100644 index 00000000000..a6d48ce8ce6 --- /dev/null +++ b/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_include_or_exclude_filters.yml.j2 @@ -0,0 +1,23 @@ +plugin: amazon.aws.aws_ec2 +aws_access_key_id: '{{ aws_access_key }}' +aws_secret_access_key: '{{ aws_secret_key }}' +{% if security_token | default(false) %} +aws_security_token: '{{ security_token }}' +{% endif %} +regions: +- '{{ aws_region }}' +filters: + tag:Name: + - '{{ resource_prefix }}_1' + - '{{ resource_prefix }}_3' +include_filters: +- tag:Name: + - '{{ resource_prefix }}_2' + - '{{ resource_prefix }}_4' +exclude_filters: +- tag:Name: + - '{{ resource_prefix }}_3' + - '{{ resource_prefix }}_4' +hostnames: +- tag:Name +- dns-name