[Breaking Change] Remove support for old boto SDK (not boto3/botocore) (

ansible-collections#630) [Breaking Change] Remove support for old boto SDK (not boto3/botocore) SUMMARY It is time. It's almost 3 years since the last code commit to the original boto library. We have deprecated and migrated. With 4.0.0 it is time for the last boto module and the relevant support code to be retired. Depends-On: ansible-collections#632 Depends-On: ansible-collections#639 ISSUE TYPE Feature Pull Request COMPONENT NAME plugins/modules/ec2.py plugins/module_utils/ec2.py plugins/doc_fragments/aws.py ADDITIONAL INFORMATION IMO we should branch stable-3 sooner rather than later and start the cleanup work. cc @jillr Reviewed-by: Brian Scholer <None> Reviewed-by: Jill R <None> Reviewed-by: Mark Chappell <None> Reviewed-by: Alina Buzachis <None>
alinabuzachis · Jan 28, 2022 · 13f5963 · 13f5963
1 parent 9f9c8c4
commit 13f5963
Show file tree

Hide file tree

Showing 41 changed files with 41 additions and 3,518 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -6,7 +6,7 @@ General information about setting up your Python environment, testing modules,
 Ansible coding styles, and more can be found in the [Ansible Community Guide](
 https://docs.ansible.com/ansible/latest/community/index.html).
 
-Information about boto library usage, module utils, testing, and more can be
+Information about AWS SDK library usage, module utils, testing, and more can be
 found in the [AWS Guidelines](https://docs.ansible.com/ansible/devel/dev_guide/platforms/aws_guidelines.html)
 documentation.
 
@@ -41,7 +41,7 @@ issue, or by reporting any additional information
 
 ## Pull Requests
 
-All modules MUST have integration tests for new features. Upgrading to boto3 shall be considered a feature request.  
+All modules MUST have integration tests for new features.
 Bug fixes for modules that currently have integration tests SHOULD have tests added.  
 New modules should be submitted to the [community.aws](https://github.com/ansible-collections/community.aws) collection
 and MUST have integration tests.

diff --git a/README.md b/README.md
@@ -23,7 +23,7 @@ Starting with the 2.0.0 releases of amazon.aws and community.aws, it is generall
 
 Version 3.0.0 of this collection supports `boto3 >= 1.16.0` and `botocore >= 1.19.0`
 
-Support for the original AWS SDK `boto` has been deprecated and the module_utils library code to support it will be removed in release 4.0.0.
+All support for the original AWS SDK `boto` was removed in release 4.0.0.
 
 ## Included content
 
@@ -110,7 +110,7 @@ be manually installed using pip:
 
 or:
 
-    pip install boto boto3 botocore
+    pip install boto3 botocore
 
 ## Using this collection
 

diff --git a/changelogs/fragments/630-remove-boto.yml b/changelogs/fragments/630-remove-boto.yml
@@ -0,0 +1,4 @@
+breaking_changes:
+- ec2 - The ``ec2`` module has been removed in release 4.0.0 and replaced by the ``ec2_instance`` module (https://github.com/ansible-collections/amazon.aws/pull/630).
+- module_utils - Support for the original AWS SDK aka ``boto`` has been removed, including all relevant helper functions.
+  All modules should now use the ``boto3``/``botocore`` AWS SDK (https://github.com/ansible-collections/amazon.aws/pull/630)
diff --git a/plugins/doc_fragments/aws.py b/plugins/doc_fragments/aws.py
@@ -53,7 +53,6 @@ class ModuleDocFragment(object):
   aws_ca_bundle:
     description:
       - "The location of a CA Bundle to use when validating SSL certificates."
-      - "Not used by boto 2 based modules."
       - "Note: The CA Bundle is read 'module' side and may need to be explicitly copied from the controller if not run locally."
     type: path
   validate_certs:
@@ -73,7 +72,6 @@ class ModuleDocFragment(object):
     description:
       - A dictionary to modify the botocore configuration.
       - Parameters can be found at U(https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html#botocore.config.Config).
-      - Only the 'user_agent' key is used for boto modules. See U(http://boto.cloudhackers.com/en/latest/boto_config_tut.html#boto) for more boto configuration.
     type: dict
 requirements:
   - python >= 3.6
@@ -94,10 +92,6 @@ class ModuleDocFragment(object):
     C(~/.aws/credentials)).
     See U(https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)
     for more information.
-  - Modules based on the original AWS SDK (boto) may read their default
-    configuration from different files.
-    See U(https://boto.readthedocs.io/en/latest/boto_config_tut.html) for more
-    information.
   - C(AWS_REGION) or C(EC2_REGION) can be typically be used to specify the
     AWS region, when required, but this can also be defined in the
     configuration files.

diff --git a/plugins/module_utils/ec2.py b/plugins/module_utils/ec2.py
@@ -31,7 +31,6 @@
 
 import os
 import re
-import sys
 import traceback
 
 from ansible.module_utils._text import to_native
@@ -61,15 +60,6 @@
 # Used to live here, moved into # ansible_collections.amazon.aws.plugins.module_utils.retries
 from .retries import AWSRetry  # pylint: disable=unused-import
 
-BOTO_IMP_ERR = None
-try:
-    import boto
-    import boto.ec2  # boto does weird import stuff
-    HAS_BOTO = True
-except ImportError:
-    BOTO_IMP_ERR = traceback.format_exc()
-    HAS_BOTO = False
-
 BOTO3_IMP_ERR = None
 try:
     import boto3
@@ -173,31 +163,22 @@ def ec2_argument_spec():
     return spec
 
 
-def get_aws_region(module, boto3=False):
+def get_aws_region(module, boto3=None):
     region = module.params.get('region')
 
     if region:
         return region
 
+    if not HAS_BOTO3:
+        module.fail_json(msg=missing_required_lib('boto3'), exception=BOTO3_IMP_ERR)
+
     if 'AWS_REGION' in os.environ:
         return os.environ['AWS_REGION']
     if 'AWS_DEFAULT_REGION' in os.environ:
         return os.environ['AWS_DEFAULT_REGION']
     if 'EC2_REGION' in os.environ:
         return os.environ['EC2_REGION']
 
-    if not boto3:
-        if not HAS_BOTO:
-            module.fail_json(msg=missing_required_lib('boto'), exception=BOTO_IMP_ERR)
-        # boto.config.get returns None if config not found
-        region = boto.config.get('Boto', 'aws_region')
-        if region:
-            return region
-        return boto.config.get('Boto', 'ec2_region')
-
-    if not HAS_BOTO3:
-        module.fail_json(msg=missing_required_lib('boto3'), exception=BOTO3_IMP_ERR)
-
     # here we don't need to make an additional call, will default to 'us-east-1' if the below evaluates to None.
     try:
         profile_name = module.params.get('profile')
@@ -206,7 +187,7 @@ def get_aws_region(module, boto3=False):
         return None
 
 
-def get_aws_connection_info(module, boto3=False):
+def get_aws_connection_info(module, boto3=None):
 
     # Check module args for credentials, then check environment vars
     # access_key
@@ -248,10 +229,6 @@ def get_aws_connection_info(module, boto3=False):
             access_key = os.environ['AWS_ACCESS_KEY']
         elif os.environ.get('EC2_ACCESS_KEY'):
             access_key = os.environ['EC2_ACCESS_KEY']
-        elif HAS_BOTO and boto.config.get('Credentials', 'aws_access_key_id'):
-            access_key = boto.config.get('Credentials', 'aws_access_key_id')
-        elif HAS_BOTO and boto.config.get('default', 'aws_access_key_id'):
-            access_key = boto.config.get('default', 'aws_access_key_id')
         else:
             # in case access_key came in as empty string
             access_key = None
@@ -263,10 +240,6 @@ def get_aws_connection_info(module, boto3=False):
             secret_key = os.environ['AWS_SECRET_KEY']
         elif os.environ.get('EC2_SECRET_KEY'):
             secret_key = os.environ['EC2_SECRET_KEY']
-        elif HAS_BOTO and boto.config.get('Credentials', 'aws_secret_access_key'):
-            secret_key = boto.config.get('Credentials', 'aws_secret_access_key')
-        elif HAS_BOTO and boto.config.get('default', 'aws_secret_access_key'):
-            secret_key = boto.config.get('default', 'aws_secret_access_key')
         else:
             # in case secret_key came in as empty string
             secret_key = None
@@ -278,10 +251,6 @@ def get_aws_connection_info(module, boto3=False):
             security_token = os.environ['AWS_SESSION_TOKEN']
         elif os.environ.get('EC2_SECURITY_TOKEN'):
             security_token = os.environ['EC2_SECURITY_TOKEN']
-        elif HAS_BOTO and boto.config.get('Credentials', 'aws_security_token'):
-            security_token = boto.config.get('Credentials', 'aws_security_token')
-        elif HAS_BOTO and boto.config.get('default', 'aws_security_token'):
-            security_token = boto.config.get('default', 'aws_security_token')
         else:
             # in case secret_token came in as empty string
             security_token = None
@@ -290,37 +259,21 @@ def get_aws_connection_info(module, boto3=False):
         if os.environ.get('AWS_CA_BUNDLE'):
             ca_bundle = os.environ.get('AWS_CA_BUNDLE')
 
-    if HAS_BOTO3 and boto3:
-        boto_params = dict(aws_access_key_id=access_key,
-                           aws_secret_access_key=secret_key,
-                           aws_session_token=security_token)
-
-        if profile_name:
-            boto_params = dict(aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None)
-            boto_params['profile_name'] = profile_name
+    boto_params = dict(aws_access_key_id=access_key,
+                       aws_secret_access_key=secret_key,
+                       aws_session_token=security_token)
 
-        if validate_certs and ca_bundle:
-            boto_params['verify'] = ca_bundle
-        else:
-            boto_params['verify'] = validate_certs
+    if profile_name:
+        boto_params = dict(aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None)
+        boto_params['profile_name'] = profile_name
 
+    if validate_certs and ca_bundle:
+        boto_params['verify'] = ca_bundle
     else:
-        boto_params = dict(aws_access_key_id=access_key,
-                           aws_secret_access_key=secret_key,
-                           security_token=security_token)
-
-        # only set profile_name if passed as an argument
-        if profile_name:
-            boto_params['profile_name'] = profile_name
-
-        boto_params['validate_certs'] = validate_certs
+        boto_params['verify'] = validate_certs
 
     if config is not None:
-        if HAS_BOTO3 and boto3:
-            boto_params['aws_config'] = botocore.config.Config(**config)
-        elif HAS_BOTO and not boto3:
-            if 'user_agent' in config:
-                sys.modules["boto.connection"].UserAgent = config['user_agent']
+        boto_params['aws_config'] = botocore.config.Config(**config)
 
     for param, value in boto_params.items():
         if isinstance(value, binary_type):
@@ -329,61 +282,6 @@ def get_aws_connection_info(module, boto3=False):
     return region, ec2_url, boto_params
 
 
-def get_ec2_creds(module):
-    ''' for compatibility mode with old modules that don't/can't yet
-        use ec2_connect method '''
-    region, ec2_url, boto_params = get_aws_connection_info(module)
-    return ec2_url, boto_params['aws_access_key_id'], boto_params['aws_secret_access_key'], region
-
-
-def boto_fix_security_token_in_profile(conn, profile_name):
-    ''' monkey patch for boto issue boto/boto#2100 '''
-    profile = 'profile ' + profile_name
-    if boto.config.has_option(profile, 'aws_security_token'):
-        conn.provider.set_security_token(boto.config.get(profile, 'aws_security_token'))
-    return conn
-
-
-def connect_to_aws(aws_module, region, **params):
-    try:
-        conn = aws_module.connect_to_region(region, **params)
-    except(boto.provider.ProfileNotFoundError):
-        raise AnsibleAWSError("Profile given for AWS was not found.  Please fix and retry.")
-    if not conn:
-        if region not in [aws_module_region.name for aws_module_region in aws_module.regions()]:
-            raise AnsibleAWSError("Region %s does not seem to be available for aws module %s. If the region definitely exists, you may need to upgrade "
-                                  "boto or extend with endpoints_path" % (region, aws_module.__name__))
-        else:
-            raise AnsibleAWSError("Unknown problem connecting to region %s for aws module %s." % (region, aws_module.__name__))
-    if params.get('profile_name'):
-        conn = boto_fix_security_token_in_profile(conn, params['profile_name'])
-    return conn
-
-
-def ec2_connect(module):
-
-    """ Return an ec2 connection"""
-
-    region, ec2_url, boto_params = get_aws_connection_info(module)
-
-    # If ec2_url is present use it
-    if ec2_url:
-        try:
-            ec2 = boto.connect_ec2_endpoint(ec2_url, **boto_params)
-        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
-            module.fail_json(msg=str(e))
-    # Otherwise, if we have a region specified, connect to its endpoint.
-    elif region:
-        try:
-            ec2 = connect_to_aws(boto.ec2, region, **boto_params)
-        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError, boto.provider.ProfileNotFoundError) as e:
-            module.fail_json(msg=str(e))
-    else:
-        module.fail_json(msg="Either region or ec2_url must be specified")
-
-    return ec2
-
-
 def ansible_dict_to_boto3_filter_list(filters_dict):
 
     """ Convert an Ansible dict of filters to list of dicts that boto3 can use
@@ -424,53 +322,38 @@ def ansible_dict_to_boto3_filter_list(filters_dict):
     return filters_list
 
 
-def get_ec2_security_group_ids_from_names(sec_group_list, ec2_connection, vpc_id=None, boto3=True):
+def get_ec2_security_group_ids_from_names(sec_group_list, ec2_connection, vpc_id=None, boto3=None):
 
     """ Return list of security group IDs from security group names. Note that security group names are not unique
      across VPCs.  If a name exists across multiple VPCs and no VPC ID is supplied, all matching IDs will be returned. This
      will probably lead to a boto exception if you attempt to assign both IDs to a resource so ensure you wrap the call in
      a try block
      """
 
-    def get_sg_name(sg, boto3):
-
-        if boto3:
-            return sg['GroupName']
-        else:
-            return sg.name
+    def get_sg_name(sg, boto3=None):
+        return str(sg['GroupName'])
 
-    def get_sg_id(sg, boto3):
-
-        if boto3:
-            return sg['GroupId']
-        else:
-            return sg.id
+    def get_sg_id(sg, boto3=None):
+        return str(sg['GroupId'])
 
     sec_group_id_list = []
 
     if isinstance(sec_group_list, string_types):
         sec_group_list = [sec_group_list]
 
     # Get all security groups
-    if boto3:
-        if vpc_id:
-            filters = [
-                {
-                    'Name': 'vpc-id',
-                    'Values': [
-                        vpc_id,
-                    ]
-                }
-            ]
-            all_sec_groups = ec2_connection.describe_security_groups(Filters=filters)['SecurityGroups']
-        else:
-            all_sec_groups = ec2_connection.describe_security_groups()['SecurityGroups']
+    if vpc_id:
+        filters = [
+            {
+                'Name': 'vpc-id',
+                'Values': [
+                    vpc_id,
+                ]
+            }
+        ]
+        all_sec_groups = ec2_connection.describe_security_groups(Filters=filters)['SecurityGroups']
     else:
-        if vpc_id:
-            filters = {'vpc-id': vpc_id}
-            all_sec_groups = ec2_connection.get_all_security_groups(filters=filters)
-        else:
-            all_sec_groups = ec2_connection.get_all_security_groups()
+        all_sec_groups = ec2_connection.describe_security_groups()['SecurityGroups']
 
     unmatched = set(sec_group_list).difference(str(get_sg_name(all_sg, boto3)) for all_sg in all_sec_groups)
     sec_group_name_list = list(set(sec_group_list) - set(unmatched))
@@ -482,7 +365,7 @@ def get_sg_id(sg, boto3):
         if len(still_unmatched) > 0:
             raise ValueError("The following group names are not valid: %s" % ', '.join(still_unmatched))
 
-    sec_group_id_list += [str(get_sg_id(all_sg, boto3)) for all_sg in all_sec_groups if str(get_sg_name(all_sg, boto3)) in sec_group_name_list]
+    sec_group_id_list += [get_sg_id(all_sg) for all_sg in all_sec_groups if get_sg_name(all_sg) in sec_group_name_list]
 
     return sec_group_id_list