From 24d7724d0381d6daf9b6c1c2eedb3db64e3ecbd8 Mon Sep 17 00:00:00 2001 From: jillr Date: Mon, 2 Mar 2020 22:17:00 +0000 Subject: [PATCH] New collection build --- .../collection-continuous-integration.yml | 0 galaxy.yml | 1 - meta/routing.yml | 46 ++++ plugins/module_utils/aws/acm.py | 6 +- plugins/module_utils/aws/rds.py | 5 +- plugins/module_utils/cloud.py | 220 ++++++++++++++++++ plugins/module_utils/ec2.py | 2 +- plugins/modules/cloudformation.py | 8 +- plugins/modules/ec2_ami.py | 6 +- plugins/modules/ec2_ami_info.py | 6 +- plugins/modules/ec2_eni.py | 4 +- plugins/modules/ec2_group.py | 5 +- plugins/modules/ec2_group_info.py | 5 +- plugins/modules/ec2_snapshot_info.py | 8 +- plugins/modules/ec2_vol.py | 8 +- plugins/modules/ec2_vol_info.py | 5 +- plugins/modules/ec2_vpc_dhcp_option_info.py | 11 +- plugins/modules/ec2_vpc_net.py | 10 +- plugins/modules/ec2_vpc_subnet.py | 9 +- shippable.yml | 40 ++++ tests/.gitignore | 2 + .../targets/ec2_group/tasks/main.yml | 6 +- .../targets/ec2_snapshot/tasks/main.yml | 10 +- .../targets/ec2_vol/tasks/main.yml | 3 + .../targets/ec2_vol_info/tasks/main.yml | 26 +-- .../targets/ec2_vpc_net/tasks/main.yml | 164 ++++++------- .../inventory_aws_ec2/playbooks/setup.yml | 2 + .../targets/inventory_aws_ec2/runme.sh | 2 +- .../templates/inventory_with_cache.yml | 2 +- .../templates/inventory_with_constructed.yml | 2 +- .../playbooks/populate_cache.yml | 2 + .../playbooks/test_populating_inventory.yml | 2 + ..._populating_inventory_with_constructed.yml | 2 + .../playbooks/test_refresh_inventory.yml | 2 + .../targets/inventory_aws_rds/runme.sh | 2 +- .../inventory_aws_rds/templates/inventory.j2 | 2 +- .../templates/inventory_with_cache.j2 | 2 +- .../templates/inventory_with_constructed.j2 | 4 +- tests/sanity/ignore-2.10.txt | 4 +- tests/utils/shippable/aws.sh | 34 +++ tests/utils/shippable/check_matrix.py | 120 ++++++++++ tests/utils/shippable/sanity.sh | 27 +++ tests/utils/shippable/shippable.sh | 142 +++++++++++ tests/utils/shippable/timing.py | 16 ++ tests/utils/shippable/timing.sh | 5 + 45 files changed, 853 insertions(+), 137 deletions(-) rename .github/{workflows => disabled-workflows}/collection-continuous-integration.yml (100%) create mode 100644 meta/routing.yml create mode 100644 plugins/module_utils/cloud.py create mode 100644 shippable.yml create mode 100755 tests/utils/shippable/aws.sh create mode 100755 tests/utils/shippable/check_matrix.py create mode 100755 tests/utils/shippable/sanity.sh create mode 100755 tests/utils/shippable/shippable.sh create mode 100755 tests/utils/shippable/timing.py create mode 100755 tests/utils/shippable/timing.sh diff --git a/.github/workflows/collection-continuous-integration.yml b/.github/disabled-workflows/collection-continuous-integration.yml similarity index 100% rename from .github/workflows/collection-continuous-integration.yml rename to .github/disabled-workflows/collection-continuous-integration.yml diff --git a/galaxy.yml b/galaxy.yml index 9138e393690..5671b0e17f0 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -9,7 +9,6 @@ license_file: COPYING tags: null dependencies: ansible.netcommon: '>=0.1.0' - community.general: '>=0.1.0' repository: git@github.com:ansible-collection-migration/ansible.amazon.git documentation: https://github.com/ansible-collection-migration/ansible.amazon/tree/master/docs homepage: https://github.com/ansible-collection-migration/ansible.amazon diff --git a/meta/routing.yml b/meta/routing.yml new file mode 100644 index 00000000000..71da35b9e73 --- /dev/null +++ b/meta/routing.yml @@ -0,0 +1,46 @@ +plugin_routing: + modules: + aws_az_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + aws_caller_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + cloudformation_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + ec2_ami_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + ec2_eni_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + ec2_group_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + ec2_snapshot_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + ec2_vol_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + ec2_vpc_dhcp_option_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + ec2_vpc_net_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details + ec2_vpc_subnet_facts: + deprecation: + removal_date: TBD + warning_text: see plugin documentation for details diff --git a/plugins/module_utils/aws/acm.py b/plugins/module_utils/aws/acm.py index 7f697f3d1e8..17a4dcae996 100644 --- a/plugins/module_utils/aws/acm.py +++ b/plugins/module_utils/aws/acm.py @@ -31,7 +31,11 @@ """ import traceback from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import get_aws_connection_info, boto3_conn -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import camel_dict_to_snake_dict, AWSRetry, HAS_BOTO3, boto3_tag_list_to_ansible_dict, ansible_dict_to_boto3_tag_list +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (camel_dict_to_snake_dict, + AWSRetry, + HAS_BOTO3, + boto3_tag_list_to_ansible_dict, + ansible_dict_to_boto3_tag_list) from ansible.module_utils._text import to_bytes diff --git a/plugins/module_utils/aws/rds.py b/plugins/module_utils/aws/rds.py index e07db9a082b..c11a51c5d27 100644 --- a/plugins/module_utils/aws/rds.py +++ b/plugins/module_utils/aws/rds.py @@ -7,7 +7,10 @@ from ansible.module_utils._text import to_text from ansible_collections.ansible.amazon.plugins.module_utils.aws.waiters import get_waiter from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import compare_aws_tags, AWSRetry, ansible_dict_to_boto3_tag_list, boto3_tag_list_to_ansible_dict +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (compare_aws_tags, + AWSRetry, + ansible_dict_to_boto3_tag_list, + boto3_tag_list_to_ansible_dict) try: from botocore.exceptions import BotoCoreError, ClientError, WaiterError diff --git a/plugins/module_utils/cloud.py b/plugins/module_utils/cloud.py new file mode 100644 index 00000000000..8d5eeba305e --- /dev/null +++ b/plugins/module_utils/cloud.py @@ -0,0 +1,220 @@ +# +# (c) 2016 Allen Sanabria, +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +# +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +""" +This module adds shared support for generic cloud modules + +In order to use this module, include it as part of a custom +module as shown below. + +from ansible.module_utils.cloud import CloudRetry + +The 'cloud' module provides the following common classes: + + * CloudRetry + - The base class to be used by other cloud providers, in order to + provide a backoff/retry decorator based on status codes. + + - Example using the AWSRetry class which inherits from CloudRetry. + + @AWSRetry.exponential_backoff(retries=10, delay=3) + get_ec2_security_group_ids_from_names() + + @AWSRetry.jittered_backoff() + get_ec2_security_group_ids_from_names() + +""" +import random +from functools import wraps +import syslog +import time + + +def _exponential_backoff(retries=10, delay=2, backoff=2, max_delay=60): + """ Customizable exponential backoff strategy. + Args: + retries (int): Maximum number of times to retry a request. + delay (float): Initial (base) delay. + backoff (float): base of the exponent to use for exponential + backoff. + max_delay (int): Optional. If provided each delay generated is capped + at this amount. Defaults to 60 seconds. + Returns: + Callable that returns a generator. This generator yields durations in + seconds to be used as delays for an exponential backoff strategy. + Usage: + >>> backoff = _exponential_backoff() + >>> backoff + + >>> list(backoff()) + [2, 4, 8, 16, 32, 60, 60, 60, 60, 60] + """ + def backoff_gen(): + for retry in range(0, retries): + sleep = delay * backoff ** retry + yield sleep if max_delay is None else min(sleep, max_delay) + return backoff_gen + + +def _full_jitter_backoff(retries=10, delay=3, max_delay=60, _random=random): + """ Implements the "Full Jitter" backoff strategy described here + https://www.awsarchitectureblog.com/2015/03/backoff.html + Args: + retries (int): Maximum number of times to retry a request. + delay (float): Approximate number of seconds to sleep for the first + retry. + max_delay (int): The maximum number of seconds to sleep for any retry. + _random (random.Random or None): Makes this generator testable by + allowing developers to explicitly pass in the a seeded Random. + Returns: + Callable that returns a generator. This generator yields durations in + seconds to be used as delays for a full jitter backoff strategy. + Usage: + >>> backoff = _full_jitter_backoff(retries=5) + >>> backoff + + >>> list(backoff()) + [3, 6, 5, 23, 38] + >>> list(backoff()) + [2, 1, 6, 6, 31] + """ + def backoff_gen(): + for retry in range(0, retries): + yield _random.randint(0, min(max_delay, delay * 2 ** retry)) + return backoff_gen + + +class CloudRetry(object): + """ CloudRetry can be used by any cloud provider, in order to implement a + backoff algorithm/retry effect based on Status Code from Exceptions. + """ + # This is the base class of the exception. + # AWS Example botocore.exceptions.ClientError + base_class = None + + @staticmethod + def status_code_from_exception(error): + """ Return the status code from the exception object + Args: + error (object): The exception itself. + """ + pass + + @staticmethod + def found(response_code, catch_extra_error_codes=None): + """ Return True if the Response Code to retry on was found. + Args: + response_code (str): This is the Response Code that is being matched against. + """ + pass + + @classmethod + def _backoff(cls, backoff_strategy, catch_extra_error_codes=None): + """ Retry calling the Cloud decorated function using the provided + backoff strategy. + Args: + backoff_strategy (callable): Callable that returns a generator. The + generator should yield sleep times for each retry of the decorated + function. + """ + def deco(f): + @wraps(f) + def retry_func(*args, **kwargs): + for delay in backoff_strategy(): + try: + return f(*args, **kwargs) + except Exception as e: + if isinstance(e, cls.base_class): + response_code = cls.status_code_from_exception(e) + if cls.found(response_code, catch_extra_error_codes): + msg = "{0}: Retrying in {1} seconds...".format(str(e), delay) + syslog.syslog(syslog.LOG_INFO, msg) + time.sleep(delay) + else: + # Return original exception if exception is not a ClientError + raise e + else: + # Return original exception if exception is not a ClientError + raise e + return f(*args, **kwargs) + + return retry_func # true decorator + + return deco + + @classmethod + def exponential_backoff(cls, retries=10, delay=3, backoff=2, max_delay=60, catch_extra_error_codes=None): + """ + Retry calling the Cloud decorated function using an exponential backoff. + + Kwargs: + retries (int): Number of times to retry a failed request before giving up + default=10 + delay (int or float): Initial delay between retries in seconds + default=3 + backoff (int or float): backoff multiplier e.g. value of 2 will + double the delay each retry + default=1.1 + max_delay (int or None): maximum amount of time to wait between retries. + default=60 + """ + return cls._backoff(_exponential_backoff( + retries=retries, delay=delay, backoff=backoff, max_delay=max_delay), catch_extra_error_codes) + + @classmethod + def jittered_backoff(cls, retries=10, delay=3, max_delay=60, catch_extra_error_codes=None): + """ + Retry calling the Cloud decorated function using a jittered backoff + strategy. More on this strategy here: + + https://www.awsarchitectureblog.com/2015/03/backoff.html + + Kwargs: + retries (int): Number of times to retry a failed request before giving up + default=10 + delay (int): Initial delay between retries in seconds + default=3 + max_delay (int): maximum amount of time to wait between retries. + default=60 + """ + return cls._backoff(_full_jitter_backoff( + retries=retries, delay=delay, max_delay=max_delay), catch_extra_error_codes) + + @classmethod + def backoff(cls, tries=10, delay=3, backoff=1.1, catch_extra_error_codes=None): + """ + Retry calling the Cloud decorated function using an exponential backoff. + + Compatibility for the original implementation of CloudRetry.backoff that + did not provide configurable backoff strategies. Developers should use + CloudRetry.exponential_backoff instead. + + Kwargs: + tries (int): Number of times to try (not retry) before giving up + default=10 + delay (int or float): Initial delay between retries in seconds + default=3 + backoff (int or float): backoff multiplier e.g. value of 2 will + double the delay each retry + default=1.1 + """ + return cls.exponential_backoff( + retries=tries - 1, delay=delay, backoff=backoff, max_delay=None, catch_extra_error_codes=catch_extra_error_codes) diff --git a/plugins/module_utils/ec2.py b/plugins/module_utils/ec2.py index 9973cf3d1e3..4115d12934d 100644 --- a/plugins/module_utils/ec2.py +++ b/plugins/module_utils/ec2.py @@ -37,7 +37,7 @@ from ansible.module_utils.ansible_release import __version__ from ansible.module_utils.basic import missing_required_lib, env_fallback from ansible.module_utils._text import to_native, to_text -from ansible_collections.community.general.plugins.module_utils.cloud import CloudRetry +from ansible_collections.ansible.amazon.plugins.module_utils.cloud import CloudRetry from ansible.module_utils.six import string_types, binary_type, text_type from ansible.module_utils.common.dict_transformations import ( camel_dict_to_snake_dict, snake_dict_to_camel_dict, diff --git a/plugins/modules/cloudformation.py b/plugins/modules/cloudformation.py index 914b2eda7c3..c6a75e94568 100644 --- a/plugins/modules/cloudformation.py +++ b/plugins/modules/cloudformation.py @@ -334,7 +334,13 @@ except ImportError: HAS_BOTO3 = False -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list, AWSRetry, boto3_conn, boto_exception, ec2_argument_spec, get_aws_connection_info +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_tag_list, + AWSRetry, + boto3_conn, + boto_exception, + ec2_argument_spec, + get_aws_connection_info, + ) from ansible.module_utils.basic import AnsibleModule from ansible.module_utils._text import to_bytes, to_native diff --git a/plugins/modules/ec2_ami.py b/plugins/modules/ec2_ami.py index a08285ec7a2..d9f8b641b65 100644 --- a/plugins/modules/ec2_ami.py +++ b/plugins/modules/ec2_ami.py @@ -351,7 +351,11 @@ ''' import time -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list, boto3_tag_list_to_ansible_dict, camel_dict_to_snake_dict, compare_aws_tags +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_tag_list, + boto3_tag_list_to_ansible_dict, + camel_dict_to_snake_dict, + compare_aws_tags, + ) from ansible_collections.ansible.amazon.plugins.module_utils.aws.core import AnsibleAWSModule try: diff --git a/plugins/modules/ec2_ami_info.py b/plugins/modules/ec2_ami_info.py index a96e045c7ed..491d286c3b9 100644 --- a/plugins/modules/ec2_ami_info.py +++ b/plugins/modules/ec2_ami_info.py @@ -208,7 +208,11 @@ pass # caught by AnsibleAWSModule from ansible_collections.ansible.amazon.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list, camel_dict_to_snake_dict, boto3_tag_list_to_ansible_dict +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_tag_list, + camel_dict_to_snake_dict, + boto3_tag_list_to_ansible_dict, + ansible_dict_to_boto3_filter_list, + ) def list_ec2_images(ec2_client, module): diff --git a/plugins/modules/ec2_eni.py b/plugins/modules/ec2_eni.py index d014944fe5f..3ce4be7c042 100644 --- a/plugins/modules/ec2_eni.py +++ b/plugins/modules/ec2_eni.py @@ -265,8 +265,8 @@ from ansible.module_utils.basic import AnsibleModule from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (AnsibleAWSError, connect_to_aws, - ec2_argument_spec, get_aws_connection_info, - get_ec2_security_group_ids_from_names) + ec2_argument_spec, get_aws_connection_info, + get_ec2_security_group_ids_from_names) def get_eni_info(interface): diff --git a/plugins/modules/ec2_group.py b/plugins/modules/ec2_group.py index ca990d3542e..8b278a56d39 100644 --- a/plugins/modules/ec2_group.py +++ b/plugins/modules/ec2_group.py @@ -402,7 +402,10 @@ from ansible_collections.ansible.amazon.plugins.module_utils.aws.iam import get_aws_account_id from ansible_collections.ansible.amazon.plugins.module_utils.aws.waiters import get_waiter from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import AWSRetry, camel_dict_to_snake_dict, compare_aws_tags -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list, boto3_tag_list_to_ansible_dict, ansible_dict_to_boto3_tag_list +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, + boto3_tag_list_to_ansible_dict, + ansible_dict_to_boto3_tag_list, + ) from ansible.module_utils.common.network import to_ipv6_subnet, to_subnet from ansible_collections.ansible.netcommon.plugins.module_utils.compat.ipaddress import ip_network, IPv6Network from ansible.module_utils._text import to_text diff --git a/plugins/modules/ec2_group_info.py b/plugins/modules/ec2_group_info.py index 18d05d1e31d..dc806a5c335 100644 --- a/plugins/modules/ec2_group_info.py +++ b/plugins/modules/ec2_group_info.py @@ -101,7 +101,10 @@ pass # caught by AnsibleAWSModule from ansible_collections.ansible.amazon.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (boto3_tag_list_to_ansible_dict, ansible_dict_to_boto3_filter_list, camel_dict_to_snake_dict) +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (boto3_tag_list_to_ansible_dict, + ansible_dict_to_boto3_filter_list, + camel_dict_to_snake_dict, + ) def main(): diff --git a/plugins/modules/ec2_snapshot_info.py b/plugins/modules/ec2_snapshot_info.py index 723ef148ca9..01df2d94f8c 100644 --- a/plugins/modules/ec2_snapshot_info.py +++ b/plugins/modules/ec2_snapshot_info.py @@ -187,8 +187,12 @@ from ansible.module_utils.basic import AnsibleModule from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, - boto3_conn, boto3_tag_list_to_ansible_dict, camel_dict_to_snake_dict, - ec2_argument_spec, get_aws_connection_info) + boto3_conn, + boto3_tag_list_to_ansible_dict, + camel_dict_to_snake_dict, + ec2_argument_spec, + get_aws_connection_info, + ) def list_ec2_snapshots(connection, module): diff --git a/plugins/modules/ec2_vol.py b/plugins/modules/ec2_vol.py index b42137d3136..770fad33458 100644 --- a/plugins/modules/ec2_vol.py +++ b/plugins/modules/ec2_vol.py @@ -238,8 +238,12 @@ pass # Taken care of by ec2.HAS_BOTO from ansible.module_utils.basic import AnsibleModule -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (HAS_BOTO, AnsibleAWSError, connect_to_aws, ec2_argument_spec, - get_aws_connection_info) +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (HAS_BOTO, + AnsibleAWSError, + connect_to_aws, + ec2_argument_spec, + get_aws_connection_info, + ) def get_volume(module, ec2): diff --git a/plugins/modules/ec2_vol_info.py b/plugins/modules/ec2_vol_info.py index cc1abc17b24..d9669529d09 100644 --- a/plugins/modules/ec2_vol_info.py +++ b/plugins/modules/ec2_vol_info.py @@ -68,7 +68,10 @@ from ansible_collections.ansible.amazon.plugins.module_utils.aws.core import AnsibleAWSModule from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import AWSRetry -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict, ansible_dict_to_boto3_filter_list, camel_dict_to_snake_dict +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (boto3_tag_list_to_ansible_dict, + ansible_dict_to_boto3_filter_list, + camel_dict_to_snake_dict, + ) def get_volume_info(volume, region): diff --git a/plugins/modules/ec2_vpc_dhcp_option_info.py b/plugins/modules/ec2_vpc_dhcp_option_info.py index b6423521280..ff8a98cfd48 100644 --- a/plugins/modules/ec2_vpc_dhcp_option_info.py +++ b/plugins/modules/ec2_vpc_dhcp_option_info.py @@ -90,9 +90,14 @@ pass # caught by imported HAS_BOTO3 from ansible.module_utils.basic import AnsibleModule -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ec2_argument_spec, boto3_conn, HAS_BOTO3, - ansible_dict_to_boto3_filter_list, get_aws_connection_info, - camel_dict_to_snake_dict, boto3_tag_list_to_ansible_dict) +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ec2_argument_spec, + boto3_conn, + HAS_BOTO3, + ansible_dict_to_boto3_filter_list, + get_aws_connection_info, + camel_dict_to_snake_dict, + boto3_tag_list_to_ansible_dict, + ) def get_dhcp_options_info(dhcp_option): diff --git a/plugins/modules/ec2_vpc_net.py b/plugins/modules/ec2_vpc_net.py index 00ae62e97f6..1eda664c938 100644 --- a/plugins/modules/ec2_vpc_net.py +++ b/plugins/modules/ec2_vpc_net.py @@ -201,11 +201,15 @@ from time import sleep, time from ansible_collections.ansible.amazon.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (AWSRetry, camel_dict_to_snake_dict, compare_aws_tags, - ansible_dict_to_boto3_tag_list, boto3_tag_list_to_ansible_dict) +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (AWSRetry, + camel_dict_to_snake_dict, + compare_aws_tags, + ansible_dict_to_boto3_tag_list, + boto3_tag_list_to_ansible_dict, + ) from ansible.module_utils.six import string_types from ansible.module_utils._text import to_native -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.utils import to_subnet +from ansible.module_utils.common.network import to_subnet def vpc_exists(module, vpc, name, cidr_block, multi): diff --git a/plugins/modules/ec2_vpc_subnet.py b/plugins/modules/ec2_vpc_subnet.py index 18aa2dce068..e84582c6d37 100644 --- a/plugins/modules/ec2_vpc_subnet.py +++ b/plugins/modules/ec2_vpc_subnet.py @@ -217,8 +217,13 @@ from ansible.module_utils._text import to_text from ansible_collections.ansible.amazon.plugins.module_utils.aws.core import AnsibleAWSModule from ansible_collections.ansible.amazon.plugins.module_utils.aws.waiters import get_waiter -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, ansible_dict_to_boto3_tag_list, - camel_dict_to_snake_dict, boto3_tag_list_to_ansible_dict, compare_aws_tags, AWSRetry) +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, + ansible_dict_to_boto3_tag_list, + camel_dict_to_snake_dict, + boto3_tag_list_to_ansible_dict, + compare_aws_tags, + AWSRetry, + ) def get_subnet_info(subnet): diff --git a/shippable.yml b/shippable.yml new file mode 100644 index 00000000000..92c37936c2b --- /dev/null +++ b/shippable.yml @@ -0,0 +1,40 @@ +language: python + +env: + matrix: + - T=none + +matrix: + exclude: + - env: T=none + include: + - env: T=sanity/1 + + - env: T=aws/2.7/1 + - env: T=aws/3.7/1 + + - env: T=aws/2.7/2 + - env: T=aws/3.7/2 + + - env: T=aws/2.7/3 + - env: T=aws/3.7/3 + + - env: T=aws/2.7/4 + - env: T=aws/3.7/4 +branches: + except: + - "*-patch-*" + - "revert-*-*" + +build: + ci: + - tests/utils/shippable/timing.sh tests/utils/shippable/shippable.sh $T + +integrations: + notifications: + - integrationName: email + type: email + on_success: never + on_failure: never + on_start: never + on_pull_request: never diff --git a/tests/.gitignore b/tests/.gitignore index ea1472ec1f3..e17f1f38fe4 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1 +1,3 @@ output/ +integration/cloud-config-aws.ini +integration/cloud-config-aws.yml diff --git a/tests/integration/targets/ec2_group/tasks/main.yml b/tests/integration/targets/ec2_group/tasks/main.yml index 9b558656cd1..6ab96df6dec 100644 --- a/tests/integration/targets/ec2_group/tasks/main.yml +++ b/tests/integration/targets/ec2_group/tasks/main.yml @@ -13,11 +13,11 @@ # EC2 Classic tests can only be run on a pre-2013 AWS account with supported-platforms=EC2 # Ansible CI does NOT have classic EC2 support; these tests are provided as-is for the # community and can be run if you have access to a classic account. To check if your account -# has support for EC2 Classic you can use the `aws_account_attribute` plugin. +# has support for EC2 Classic you can use the `ansible.amazon.aws_account_attribute` plugin. - name: determine if this is an EC2 Classic account set_fact: - has_ec2_classic: "{{ lookup('aws_account_attribute', + has_ec2_classic: "{{ lookup('ansible.amazon.aws_account_attribute', attribute='has-ec2-classic', region=aws_region, aws_access_key=aws_access_key, @@ -55,7 +55,7 @@ block: - name: determine if there is a default VPC set_fact: - defaultvpc: "{{ lookup('aws_account_attribute', + defaultvpc: "{{ lookup('ansible.amazon.aws_account_attribute', attribute='default-vpc', region=aws_region, aws_access_key=aws_access_key, diff --git a/tests/integration/targets/ec2_snapshot/tasks/main.yml b/tests/integration/targets/ec2_snapshot/tasks/main.yml index b8cdec3045c..60665559df7 100644 --- a/tests/integration/targets/ec2_snapshot/tasks/main.yml +++ b/tests/integration/targets/ec2_snapshot/tasks/main.yml @@ -20,6 +20,10 @@ security_token: "{{ security_token | default(omit) }}" region: "{{ aws_region }}" + collections: + - community.amazon + + block: - ec2_ami_info: owners: amazon @@ -144,7 +148,7 @@ that: - result is changed - info_result.snapshots| length == 2 - - '"{{ result.snapshot_id }}" in "{{ info_result| json_query("snapshots[].snapshot_id") }}"' + - '"{{ result.snapshot_id }}" in "{{ info_result| community.general.json_query("snapshots[].snapshot_id") }}"' # JR: Check mode not supported # - name: Take snapshot with a tag (check mode) @@ -210,7 +214,7 @@ - assert: that: - info_result.snapshots| length == 2 - - '"{{ tagged_snapshot_id }}" not in "{{ info_result| json_query("snapshots[].snapshot_id") }}"' + - '"{{ tagged_snapshot_id }}" not in "{{ info_result| community.general.json_query("snapshots[].snapshot_id") }}"' - name: Delete snapshots ec2_snapshot: @@ -253,4 +257,4 @@ ec2_vol: id: '{{ volume_id }}' state: absent - ignore_errors: true \ No newline at end of file + ignore_errors: true diff --git a/tests/integration/targets/ec2_vol/tasks/main.yml b/tests/integration/targets/ec2_vol/tasks/main.yml index aa81248e306..602bfb4718b 100644 --- a/tests/integration/targets/ec2_vol/tasks/main.yml +++ b/tests/integration/targets/ec2_vol/tasks/main.yml @@ -7,6 +7,9 @@ security_token: '{{ security_token | default(omit) }}' region: '{{ aws_region | default(omit) }}' + collections: + - community.amazon + block: # ==== Env setup ========================================================== diff --git a/tests/integration/targets/ec2_vol_info/tasks/main.yml b/tests/integration/targets/ec2_vol_info/tasks/main.yml index 9bde0307613..bbc798bc045 100644 --- a/tests/integration/targets/ec2_vol_info/tasks/main.yml +++ b/tests/integration/targets/ec2_vol_info/tasks/main.yml @@ -1,18 +1,18 @@ --- # tasks file for test_ec2_vol_info -- name: Set up AWS connection info - set_fact: - aws_connection_info: &aws_connection_info - aws_access_key: "{{ aws_access_key }}" - aws_secret_key: "{{ aws_secret_key }}" - security_token: "{{ security_token }}" - region: "{{ aws_region }}" - no_log: true +- module_defaults: + group/aws: + aws_access_key: '{{ aws_access_key | default(omit) }}' + aws_secret_key: '{{ aws_secret_key | default(omit) }}' + security_token: '{{ security_token | default(omit) }}' + region: '{{ aws_region | default(omit) }}' -- block: + collections: + - community.amazon + + block: - ec2_ami_info: owners: amazon - <<: *aws_connection_info filters: architecture: x86_64 virtualization-type: hvm @@ -28,7 +28,6 @@ wait: yes tags: Environment: test - <<: *aws_connection_info register: instance - name: Ensure there's only one matching instance @@ -46,13 +45,11 @@ iops: 100 tags: Tag Name with Space-and-dash: Tag Value with Space-and-dash - <<: *aws_connection_info delete_on_termination: yes register: volume - name: Gather volume info without any filters ec2_vol_info: - <<: *aws_connection_info register: volume_facts_wo_filters check_mode: no @@ -63,7 +60,6 @@ - name: Gather volume info ec2_vol_info: - <<: *aws_connection_info filters: "tag:Name": "{{ resource_prefix }}_ansible_ec2_vol_info_test.db" register: volume_facts @@ -106,7 +102,6 @@ state: absent filters: "tag:Name": "{{ resource_prefix }}_ansible_ec2_vol_info_test" - <<: *aws_connection_info register: result until: result is not failed ignore_errors: yes @@ -116,7 +111,6 @@ ec2_vol: id: "{{ volume.volume_id }}" state: absent - <<: *aws_connection_info register: result until: result is not failed ignore_errors: yes diff --git a/tests/integration/targets/ec2_vpc_net/tasks/main.yml b/tests/integration/targets/ec2_vpc_net/tasks/main.yml index c0e5e1b45f0..f45bb08316c 100644 --- a/tests/integration/targets/ec2_vpc_net/tasks/main.yml +++ b/tests/integration/targets/ec2_vpc_net/tasks/main.yml @@ -110,7 +110,7 @@ - '"instance_tenancy" in result.vpc' - result.vpc.ipv6_cidr_block_association_set | length == 1 - result.vpc.ipv6_cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - - result.vpc.ipv6_cidr_block_association_set[0].ipv6_cidr_block | ipv6 + - result.vpc.ipv6_cidr_block_association_set[0].ipv6_cidr_block | ansible.netcommon.ipv6 - result.vpc.ipv6_cidr_block_association_set[0].ipv6_cidr_block_state.state in ["associated", "associating"] - '"is_default" in result.vpc' - '"state" in result.vpc' @@ -154,7 +154,7 @@ - '"instance_tenancy" in result.vpc' - result.vpc.ipv6_cidr_block_association_set | length == 1 - result.vpc.ipv6_cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - - result.vpc.ipv6_cidr_block_association_set[0].ipv6_cidr_block | ipv6 + - result.vpc.ipv6_cidr_block_association_set[0].ipv6_cidr_block | ansible.netcommon.ipv6 - result.vpc.ipv6_cidr_block_association_set[0].ipv6_cidr_block_state.state in ["associated", "associating"] - '"is_default" in result.vpc' - '"state" in result.vpc' @@ -171,7 +171,7 @@ - name: Test that our new VPC shows up in the results assert: that: - - vpc_1 in ( vpc_info | json_query("vpcs[].vpc_id") | list ) + - vpc_1 in ( vpc_info | community.general.json_query("vpcs[].vpc_id") | list ) - name: VPC info (Simple tag filter) ec2_vpc_net_info: @@ -781,17 +781,17 @@ # - result.vpc.id == vpc_1 # - vpc_info.vpcs | length == 1 # - vpc_info.vpcs[0].cidr_block == vpc_cidr - # - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a not in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b not in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_a not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) # - vpc_info.vpcs[0].cidr_block_association_set | length == 1 # - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") # - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") # - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] # - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - # - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a not in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b not in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_a not in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - name: modify CIDR ec2_vpc_net: @@ -820,17 +820,17 @@ - result.vpc.cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b not in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 2 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b not in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b not in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - name: modify CIDR (no change) ec2_vpc_net: @@ -859,17 +859,17 @@ - result.vpc.cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b not in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 2 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b not in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b not in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) # #62678 #- name: modify CIDR - no purge (check mode) @@ -893,17 +893,17 @@ # - result is changed # - vpc_info.vpcs | length == 1 # - vpc_info.vpcs[0].cidr_block == vpc_cidr - # - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b not in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) # - vpc_info.vpcs[0].cidr_block_association_set | length == 2 # - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") # - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") # - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] # - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - # - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b not in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_b not in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - name: modify CIDR - no purge ec2_vpc_net: @@ -934,9 +934,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -944,9 +944,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - name: modify CIDR - no purge (no change) ec2_vpc_net: @@ -976,9 +976,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -986,9 +986,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - name: modify CIDR - no purge (no change - list all - check mode) ec2_vpc_net: @@ -1019,9 +1019,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -1029,9 +1029,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - name: modify CIDR - no purge (no change - list all) ec2_vpc_net: @@ -1062,9 +1062,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -1072,9 +1072,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - name: modify CIDR - no purge (no change - different order - check mode) ec2_vpc_net: @@ -1105,9 +1105,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -1115,9 +1115,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - name: modify CIDR - no purge (no change - different order) ec2_vpc_net: @@ -1148,9 +1148,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -1158,9 +1158,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) # #62678 #- name: modify CIDR - purge (check mode) @@ -1192,9 +1192,9 @@ # - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] # - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] # - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - # - vpc_cidr in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b in (vpc_info.vpcs[0] | json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - name: modify CIDR - purge ec2_vpc_net: @@ -1221,14 +1221,14 @@ - vpc_info.vpcs | length == 1 - result.vpc.cidr_block == vpc_cidr - vpc_info.vpcs[0].cidr_block == vpc_cidr - - result.vpc | json_query(cidr_query) | list | length == 2 - - vpc_cidr in (result.vpc | json_query(cidr_query) | list) - - vpc_cidr_a not in (result.vpc | json_query(cidr_query) | list) - - vpc_cidr_b in (result.vpc | json_query(cidr_query) | list) - - vpc_info.vpcs[0] | json_query(cidr_query) | list | length == 2 - - vpc_cidr in (vpc_info.vpcs[0] | json_query(cidr_query) | list) - - vpc_cidr_a not in (vpc_info.vpcs[0] | json_query(cidr_query) | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | json_query(cidr_query) | list) + - result.vpc | community.general.json_query(cidr_query) | list | length == 2 + - vpc_cidr in (result.vpc | community.general.json_query(cidr_query) | list) + - vpc_cidr_a not in (result.vpc | community.general.json_query(cidr_query) | list) + - vpc_cidr_b in (result.vpc | community.general.json_query(cidr_query) | list) + - vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list | length == 2 + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) + - vpc_cidr_a not in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) + - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) - name: modify CIDR - purge (no change) ec2_vpc_net: @@ -1255,14 +1255,14 @@ - vpc_info.vpcs | length == 1 - result.vpc.cidr_block == vpc_cidr - vpc_info.vpcs[0].cidr_block == vpc_cidr - - result.vpc | json_query(cidr_query) | list | length == 2 - - vpc_cidr in (result.vpc | json_query(cidr_query) | list) - - vpc_cidr_a not in (result.vpc | json_query(cidr_query) | list) - - vpc_cidr_b in (result.vpc | json_query(cidr_query) | list) - - vpc_info.vpcs[0] | json_query(cidr_query) | list | length == 2 - - vpc_cidr in (vpc_info.vpcs[0] | json_query(cidr_query) | list) - - vpc_cidr_a not in (vpc_info.vpcs[0] | json_query(cidr_query) | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | json_query(cidr_query) | list) + - result.vpc | community.general.json_query(cidr_query) | list | length == 2 + - vpc_cidr in (result.vpc | community.general.json_query(cidr_query) | list) + - vpc_cidr_a not in (result.vpc | community.general.json_query(cidr_query) | list) + - vpc_cidr_b in (result.vpc | community.general.json_query(cidr_query) | list) + - vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list | length == 2 + - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) + - vpc_cidr_a not in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) + - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) # ============================================================ diff --git a/tests/integration/targets/inventory_aws_ec2/playbooks/setup.yml b/tests/integration/targets/inventory_aws_ec2/playbooks/setup.yml index 8a9b88937f1..d166beb2724 100644 --- a/tests/integration/targets/inventory_aws_ec2/playbooks/setup.yml +++ b/tests/integration/targets/inventory_aws_ec2/playbooks/setup.yml @@ -6,6 +6,8 @@ security_token: '{{ security_token }}' region: '{{ aws_region }}' no_log: yes + collections: + - community.amazon - name: get image ID to create an instance ec2_ami_info: diff --git a/tests/integration/targets/inventory_aws_ec2/runme.sh b/tests/integration/targets/inventory_aws_ec2/runme.sh index 916f7e8f7ae..cd73f6a2c39 100755 --- a/tests/integration/targets/inventory_aws_ec2/runme.sh +++ b/tests/integration/targets/inventory_aws_ec2/runme.sh @@ -5,7 +5,7 @@ set -eux # ensure test config is empty ansible-playbook playbooks/empty_inventory_config.yml "$@" -export ANSIBLE_INVENTORY_ENABLED=aws_ec2 +export ANSIBLE_INVENTORY_ENABLED="ansible.amazon.aws_ec2" # test with default inventory file ansible-playbook playbooks/test_invalid_aws_ec2_inventory_config.yml "$@" diff --git a/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_cache.yml b/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_cache.yml index 786174281c6..baf4a549674 100644 --- a/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_cache.yml +++ b/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_cache.yml @@ -1,6 +1,6 @@ plugin: ansible.amazon.aws_ec2 cache: true -cache_plugin: community.general.jsonfile +cache_plugin: jsonfile cache_connection: aws_ec2_cache_dir aws_access_key_id: '{{ aws_access_key }}' aws_secret_access_key: '{{ aws_secret_key }}' diff --git a/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_constructed.yml b/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_constructed.yml index 902ee91391e..17fe046f9a8 100644 --- a/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_constructed.yml +++ b/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_constructed.yml @@ -8,7 +8,7 @@ filters: tag:Name: - '{{ resource_prefix }}' keyed_groups: -- key: security_groups|community.general.json_query("[].group_id") +- key: 'security_groups|community.general.json_query("[].group_id")' prefix: security_groups - key: tags prefix: tag diff --git a/tests/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml b/tests/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml index bd7dc6b4944..0b672ac2b9d 100644 --- a/tests/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml +++ b/tests/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml @@ -3,6 +3,8 @@ connection: local gather_facts: no environment: "{{ ansible_test.environment }}" + collections: + - community.amazon tasks: - module_defaults: diff --git a/tests/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml b/tests/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml index d79f2a01a9a..93df84dc7ea 100644 --- a/tests/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml +++ b/tests/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml @@ -3,6 +3,8 @@ connection: local gather_facts: no environment: "{{ ansible_test.environment }}" + collections: + - community.amazon tasks: - module_defaults: diff --git a/tests/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml b/tests/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml index c6ddb57340c..0ed8589fa07 100644 --- a/tests/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml +++ b/tests/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml @@ -3,6 +3,8 @@ connection: local gather_facts: no environment: "{{ ansible_test.environment }}" + collections: + - community.amazon tasks: - module_defaults: diff --git a/tests/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml b/tests/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml index 565803800c8..3fce0c944e8 100644 --- a/tests/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml +++ b/tests/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml @@ -5,6 +5,8 @@ aws_secret_key: '{{ aws_secret_key }}' security_token: '{{ security_token | default(omit) }}' region: '{{ aws_region }}' + collections: + - community.amazon block: - set_fact: instance_id: "{{ resource_prefix }}update" diff --git a/tests/integration/targets/inventory_aws_rds/runme.sh b/tests/integration/targets/inventory_aws_rds/runme.sh index d759349e769..797cd1ea433 100755 --- a/tests/integration/targets/inventory_aws_rds/runme.sh +++ b/tests/integration/targets/inventory_aws_rds/runme.sh @@ -5,7 +5,7 @@ set -eux # ensure test config is empty ansible-playbook playbooks/empty_inventory_config.yml "$@" -export ANSIBLE_INVENTORY_ENABLED=aws_rds +export ANSIBLE_INVENTORY_ENABLED="ansible.amazon.aws_rds" # test with default inventory file ansible-playbook playbooks/test_invalid_aws_rds_inventory_config.yml "$@" diff --git a/tests/integration/targets/inventory_aws_rds/templates/inventory.j2 b/tests/integration/targets/inventory_aws_rds/templates/inventory.j2 index 3d9df9affcf..cb29284890d 100644 --- a/tests/integration/targets/inventory_aws_rds/templates/inventory.j2 +++ b/tests/integration/targets/inventory_aws_rds/templates/inventory.j2 @@ -1,4 +1,4 @@ -plugin: aws_rds +plugin: ansible.amazon.aws_rds aws_access_key_id: '{{ aws_access_key }}' aws_secret_access_key: '{{ aws_secret_key }}' {% if security_token | default(false) %} diff --git a/tests/integration/targets/inventory_aws_rds/templates/inventory_with_cache.j2 b/tests/integration/targets/inventory_aws_rds/templates/inventory_with_cache.j2 index ba227e3082d..b68ffc72c70 100644 --- a/tests/integration/targets/inventory_aws_rds/templates/inventory_with_cache.j2 +++ b/tests/integration/targets/inventory_aws_rds/templates/inventory_with_cache.j2 @@ -1,4 +1,4 @@ -plugin: aws_rds +plugin: ansible.amazon.aws_rds cache: True cache_plugin: jsonfile cache_connection: aws_rds_cache_dir diff --git a/tests/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.j2 b/tests/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.j2 index 72394974784..2b877f9ba1e 100644 --- a/tests/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.j2 +++ b/tests/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.j2 @@ -1,4 +1,4 @@ -plugin: aws_rds +plugin: ansible.amazon.aws_rds aws_access_key_id: '{{ aws_access_key }}' aws_secret_access_key: '{{ aws_secret_key }}' {% if security_token | default(false) %} @@ -7,7 +7,7 @@ aws_security_token: '{{ security_token }}' regions: - '{{ aws_region }}' keyed_groups: - - key: 'db_parameter_groups|json_query("[].db_parameter_group_name")' + - key: 'db_parameter_groups|community.general.json_query("[].db_parameter_group_name")' prefix: rds_parameter_group - key: tags prefix: tag diff --git a/tests/sanity/ignore-2.10.txt b/tests/sanity/ignore-2.10.txt index 99c5f548f45..3d34f5c616d 100644 --- a/tests/sanity/ignore-2.10.txt +++ b/tests/sanity/ignore-2.10.txt @@ -44,4 +44,6 @@ tests/unit/module_utils/conftest.py metaclass-boilerplate tests/unit/modules/conftest.py future-import-boilerplate tests/unit/modules/conftest.py metaclass-boilerplate tests/unit/modules/utils.py future-import-boilerplate -tests/unit/modules/utils.py metaclass-boilerplate \ No newline at end of file +tests/unit/modules/utils.py metaclass-boilerplate +tests/utils/shippable/check_matrix.py replace-urlopen +tests/utils/shippable/timing.py shebang diff --git a/tests/utils/shippable/aws.sh b/tests/utils/shippable/aws.sh new file mode 100755 index 00000000000..8c613a9b3a3 --- /dev/null +++ b/tests/utils/shippable/aws.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -o pipefail -eux + +declare -a args +IFS='/:' read -ra args <<< "$1" + +cloud="${args[0]}" +python="${args[1]}" +group="${args[2]}" + +target="shippable/${cloud}/group${group}/" + +stage="${S:-prod}" + +changed_all_target="shippable/${cloud}/smoketest/" + +if [ "${group}" == "1" ]; then + # only run smoketest tests for group1 + changed_all_mode="include" + + if ! ansible-test integration "${changed_all_target}" --list-targets > /dev/null 2>&1; then + # no smoketest tests are available for this cloud + changed_all_target="none" + fi +else + # smoketest tests already covered by group1 + changed_all_mode="exclude" +fi + +# shellcheck disable=SC2086 +ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \ + --remote-terminate always --remote-stage "${stage}" \ + --docker --python "${python}" --changed-all-target "${changed_all_target}" --changed-all-mode "${changed_all_mode}" diff --git a/tests/utils/shippable/check_matrix.py b/tests/utils/shippable/check_matrix.py new file mode 100755 index 00000000000..1e555198499 --- /dev/null +++ b/tests/utils/shippable/check_matrix.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +"""Verify the currently executing Shippable test matrix matches the one defined in the "shippable.yml" file.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import datetime +import json +import os +import re +import sys +import time + +try: + from typing import NoReturn +except ImportError: + NoReturn = None + +try: + # noinspection PyCompatibility + from urllib2 import urlopen # pylint: disable=ansible-bad-import-from +except ImportError: + # noinspection PyCompatibility + from urllib.request import urlopen + + +def main(): # type: () -> None + """Main entry point.""" + repo_full_name = os.environ['REPO_FULL_NAME'] + required_repo_full_name = 'ansible-collections/amazon-core' + + if repo_full_name != required_repo_full_name: + sys.stderr.write('Skipping matrix check on repo "%s" which is not "%s".\n' % (repo_full_name, required_repo_full_name)) + return + + with open('shippable.yml', 'rb') as yaml_file: + yaml = yaml_file.read().decode('utf-8').splitlines() + + defined_matrix = [match.group(1) for match in [re.search(r'^ *- env: T=(.*)$', line) for line in yaml] if match and match.group(1) != 'none'] + + if not defined_matrix: + fail('No matrix entries found in the "shippable.yml" file.', + 'Did you modify the "shippable.yml" file?') + + run_id = os.environ['SHIPPABLE_BUILD_ID'] + sleep = 1 + jobs = [] + + for attempts_remaining in range(4, -1, -1): + try: + jobs = json.loads(urlopen('https://api.shippable.com/jobs?runIds=%s' % run_id).read()) + + if not isinstance(jobs, list): + raise Exception('Shippable run %s data is not a list.' % run_id) + + break + except Exception as ex: + if not attempts_remaining: + fail('Unable to retrieve Shippable run %s matrix.' % run_id, + str(ex)) + + sys.stderr.write('Unable to retrieve Shippable run %s matrix: %s\n' % (run_id, ex)) + sys.stderr.write('Trying again in %d seconds...\n' % sleep) + time.sleep(sleep) + sleep *= 2 + + if len(jobs) != len(defined_matrix): + if len(jobs) == 1: + hint = '\n\nMake sure you do not use the "Rebuild with SSH" option.' + else: + hint = '' + + fail('Shippable run %s has %d jobs instead of the expected %d jobs.' % (run_id, len(jobs), len(defined_matrix)), + 'Try re-running the entire matrix.%s' % hint) + + actual_matrix = dict((job.get('jobNumber'), dict(tuple(line.split('=', 1)) for line in job.get('env', [])).get('T', '')) for job in jobs) + errors = [(job_number, test, actual_matrix.get(job_number)) for job_number, test in enumerate(defined_matrix, 1) if actual_matrix.get(job_number) != test] + + if len(errors): + error_summary = '\n'.join('Job %s expected "%s" but found "%s" instead.' % (job_number, expected, actual) for job_number, expected, actual in errors) + + fail('Shippable run %s has a job matrix mismatch.' % run_id, + 'Try re-running the entire matrix.\n\n%s' % error_summary) + + +def fail(message, output): # type: (str, str) -> NoReturn + # Include a leading newline to improve readability on Shippable "Tests" tab. + # Without this, the first line becomes indented. + output = '\n' + output.strip() + + timestamp = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + + # hack to avoid requiring junit-xml, which isn't pre-installed on Shippable outside our test containers + xml = ''' + + +\t +\t\t +\t\t\t%s +\t\t +\t + +''' % (timestamp, message, output) + + path = 'shippable/testresults/check-matrix.xml' + dir_path = os.path.dirname(path) + + if not os.path.exists(dir_path): + os.makedirs(dir_path) + + with open(path, 'w') as junit_fd: + junit_fd.write(xml.lstrip()) + + sys.stderr.write(message + '\n') + sys.stderr.write(output + '\n') + + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/tests/utils/shippable/sanity.sh b/tests/utils/shippable/sanity.sh new file mode 100755 index 00000000000..d5aa1fe5568 --- /dev/null +++ b/tests/utils/shippable/sanity.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -o pipefail -eux + +declare -a args +IFS='/:' read -ra args <<< "$1" + +group="${args[1]}" + +if [ "${BASE_BRANCH:-}" ]; then + base_branch="origin/${BASE_BRANCH}" +else + base_branch="" +fi + +case "${group}" in + 1) options=(--skip-test pylint --skip-test ansible-doc --skip-test validate-modules) ;; + 2) options=( --test ansible-doc) ;; + 3) options=(--test pylint --exclude test/units/ --exclude lib/ansible/module_utils/) ;; + 4) options=(--test pylint test/units/ lib/ansible/module_utils/) ;; + 5) options=( --test validate-modules) ;; +esac + +# shellcheck disable=SC2086 +ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ + --docker --base-branch "${base_branch}" \ + "${options[@]}" --allow-disabled diff --git a/tests/utils/shippable/shippable.sh b/tests/utils/shippable/shippable.sh new file mode 100755 index 00000000000..a364412469a --- /dev/null +++ b/tests/utils/shippable/shippable.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +set -o pipefail -eux + +declare -a args +IFS='/:' read -ra args <<< "$1" + +script="${args[0]}" + +test="$1" + +docker images ansible/ansible +docker images quay.io/ansible/* +docker ps + +for container in $(docker ps --format '{{.Image}} {{.ID}}' | grep -v '^drydock/' | sed 's/^.* //'); do + docker rm -f "${container}" || true # ignore errors +done + +docker ps + +if [ -d /home/shippable/cache/ ]; then + ls -la /home/shippable/cache/ +fi + +command -v python +python -V + +command -v pip +pip --version +pip list --disable-pip-version-check + +export PATH="${PWD}/bin:${PATH}" +export PYTHONIOENCODING='utf-8' + +if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then +# COVERAGE=yes + COMPLETE=yes +fi + +#if [ -n "${COVERAGE:-}" ]; then +# # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value +# export COVERAGE="--coverage" +#elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then +# # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message +# export COVERAGE="--coverage" +#else +# # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled) +# export COVERAGE="--coverage-check" +#fi + +if [ -n "${COMPLETE:-}" ]; then + # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value + export CHANGED="" +elif [[ "${COMMIT_MESSAGE}" =~ ci_complete ]]; then + # disable change detection triggered by having 'ci_complete' in the latest commit message + export CHANGED="" +else + # enable change detection (default behavior) + #export CHANGED="--changed" + export CHANGED="" # disable changed detection https://github.com/ansible/ansible/issues/67869 +fi + +if [ "${IS_PULL_REQUEST:-}" == "true" ]; then + # run unstable tests which are targeted by focused changes on PRs + export UNSTABLE="--allow-unstable-changed" +else + # do not run unstable tests outside PRs + export UNSTABLE="" +fi + +virtualenv --python /usr/bin/python3.7 ~/ansible-venv +set +ux +. ~/ansible-venv/bin/activate +set -ux + +pip install git+https://github.com/ansible-collection-migration/ansible-base + +#ansible-galaxy collection install community.general +mkdir -p "${HOME}/.ansible/ansible_collections/community" +mkdir -p "${HOME}/.ansible/ansible_collections/google" +mkdir -p "${HOME}/.ansible/ansible_collections/openstack" +cwd=$(pwd) +cd "${HOME}/.ansible/ansible_collections/" +git clone https://github.com/ansible-collections/community.general community/general +git clone https://github.com/ansible-collection-migration/community.amazon community/amazon +# community.general requires a lot of things we need to manual pull in +# once community.general is published this will be handled by galaxy cli +git clone https://github.com/ansible-collection-migration/google.cloud google/cloud +git clone https://github.com/ansible-collection-migration/openstack.cloud openstack/cloud +git clone https://github.com/ansible-collection-migration/ansible.netcommon ansible/netcommon +#ansible-galaxy collection install ansible.netcommon +cd "${cwd}" + +export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible/" +TEST_DIR="${HOME}/.ansible/ansible_collections/ansible/amazon/" +mkdir -p "${TEST_DIR}" +cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}" +cd "${TEST_DIR}" + +function cleanup +{ + if [ -d tests/output/coverage/ ]; then + if find tests/output/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then + # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then + stub="--stub" + else + stub="" + fi + + # shellcheck disable=SC2086 + ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"} + cp -a tests/output/reports/coverage=*.xml shippable/codecoverage/ + fi + fi + + if [ -d tests/output/junit/ ]; then + cp -aT tests/output/junit/ shippable/testresults/ + fi + + if [ -d tests/output/data/ ]; then + cp -a tests/output/data/ shippable/testresults/ + fi + + if [ -d tests/output/bot/ ]; then + cp -aT tests/output/bot/ shippable/testresults/ + fi +} + +trap cleanup EXIT + +if [[ "${COVERAGE:-}" == "--coverage" ]]; then + timeout=60 +else + timeout=45 +fi + +ansible-test env --dump --show --timeout "${timeout}" --color -v + +"tests/utils/shippable/check_matrix.py" +"tests/utils/shippable/${script}.sh" "${test}" diff --git a/tests/utils/shippable/timing.py b/tests/utils/shippable/timing.py new file mode 100755 index 00000000000..fb538271b84 --- /dev/null +++ b/tests/utils/shippable/timing.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3.7 +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import sys +import time + +start = time.time() + +sys.stdin.reconfigure(errors='surrogateescape') +sys.stdout.reconfigure(errors='surrogateescape') + +for line in sys.stdin: + seconds = time.time() - start + sys.stdout.write('%02d:%02d %s' % (seconds // 60, seconds % 60, line)) + sys.stdout.flush() diff --git a/tests/utils/shippable/timing.sh b/tests/utils/shippable/timing.sh new file mode 100755 index 00000000000..77e25783046 --- /dev/null +++ b/tests/utils/shippable/timing.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -o pipefail -eu + +"$@" 2>&1 | "$(dirname "$0")/timing.py"