From 46f9de93887d8dac56b2bb764410f81edab01078 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Mon, 22 Nov 2021 18:04:27 +0100 Subject: [PATCH 1/2] [3.0.0] Bump botocore version --- README.md | 2 +- plugins/doc_fragments/aws.py | 4 +- plugins/lookup/aws_account_attribute.py | 2 +- plugins/lookup/aws_secret.py | 2 +- plugins/lookup/aws_ssm.py | 2 +- plugins/module_utils/core.py | 8 +- plugins/modules/s3_bucket.py | 7 - requirements.txt | 4 +- tests/integration/constraints.txt | 6 +- .../s3_bucket/tasks/ownership_controls.yml | 198 +++++++++--------- .../setup_botocore_pip/defaults/main.yml | 4 +- tests/unit/constraints.txt | 6 +- .../test_minimal_versions.py | 12 +- 13 files changed, 120 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index 42040d9e1e6..1c3155e674e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ As the AWS SDK for Python (Boto3 and Botocore) has [ceased supporting Python 2.7 Starting with the 2.0.0 releases of amazon.aws and community.aws, it is generally the collection's policy to support the versions of `botocore` and `boto3` that were released 12 months prior to the most recent major collection release, following semantic versioning (for example, 2.0.0, 3.0.0). -Version 2.0.0 of this collection supports `boto3 >= 1.15.0` and `botocore >= 1.18.0` +Version 3.0.0 of this collection supports `boto3 >= 1.16.0` and `botocore >= 1.19.0` ## Included content diff --git a/plugins/doc_fragments/aws.py b/plugins/doc_fragments/aws.py index cad6fe0d0ea..a9b5747c13b 100644 --- a/plugins/doc_fragments/aws.py +++ b/plugins/doc_fragments/aws.py @@ -77,8 +77,8 @@ class ModuleDocFragment(object): type: dict requirements: - python >= 3.6 - - boto3 >= 1.15.0 - - botocore >= 1.18.0 + - boto3 >= 1.16.0 + - botocore >= 1.19.0 notes: - If parameters are not set within the module, the following environment variables can be used in decreasing order of precedence diff --git a/plugins/lookup/aws_account_attribute.py b/plugins/lookup/aws_account_attribute.py index 6ac58d40c90..428c09b6db8 100644 --- a/plugins/lookup/aws_account_attribute.py +++ b/plugins/lookup/aws_account_attribute.py @@ -10,7 +10,7 @@ requirements: - python >= 3.6 - boto3 - - botocore >= 1.18.0 + - botocore >= 1.19.0 extends_documentation_fragment: - amazon.aws.aws_credentials - amazon.aws.aws_region diff --git a/plugins/lookup/aws_secret.py b/plugins/lookup/aws_secret.py index 1bb815989e6..5eac1a38906 100644 --- a/plugins/lookup/aws_secret.py +++ b/plugins/lookup/aws_secret.py @@ -11,7 +11,7 @@ requirements: - python >= 3.6 - boto3 - - botocore >= 1.18.0 + - botocore >= 1.19.0 extends_documentation_fragment: - amazon.aws.aws_credentials - amazon.aws.aws_region diff --git a/plugins/lookup/aws_ssm.py b/plugins/lookup/aws_ssm.py index 678dbed851f..89cfe379465 100644 --- a/plugins/lookup/aws_ssm.py +++ b/plugins/lookup/aws_ssm.py @@ -16,7 +16,7 @@ requirements: - python >= 3.6 - boto3 - - botocore >= 1.18.0 + - botocore >= 1.19.0 short_description: Get the value for a SSM parameter or all parameters under a path. description: - Get the value for an Amazon Simple Systems Manager parameter or a hierarchy of parameters. diff --git a/plugins/module_utils/core.py b/plugins/module_utils/core.py index 74d47533395..d41e46df025 100644 --- a/plugins/module_utils/core.py +++ b/plugins/module_utils/core.py @@ -126,11 +126,11 @@ def __init__(self, **kwargs): self._module.fail_json( msg=missing_required_lib('botocore or boto3')) current_versions = self._gather_versions() - if not self.botocore_at_least('1.18.0'): - self.warn('botocore < 1.18.0 is not supported or tested.' + if not self.botocore_at_least('1.19.0'): + self.warn('botocore < 1.19.0 is not supported or tested.' ' Some features may not work.') - if not self.boto3_at_least("1.15.0"): - self.warn('boto3 < 1.15.0 is not supported or tested.' + if not self.boto3_at_least("1.16.0"): + self.warn('boto3 < 1.16.0 is not supported or tested.' ' Some features may not work.') self.check_mode = self._module.check_mode diff --git a/plugins/modules/s3_bucket.py b/plugins/modules/s3_bucket.py index 98f146a35b1..83b76f4b833 100644 --- a/plugins/modules/s3_bucket.py +++ b/plugins/modules/s3_bucket.py @@ -129,7 +129,6 @@ - C(ObjectWriter) - The uploading account will own the object if the object is uploaded with the bucket-owner-full-control canned ACL. - This option cannot be used together with a I(delete_object_ownership) definition. - - Management of bucket ownership controls requires botocore>=1.18.11. choices: [ 'BucketOwnerPreferred', 'ObjectWriter' ] type: str version_added: 2.0.0 @@ -137,7 +136,6 @@ description: - Delete bucket's ownership controls. - This option cannot be used together with a I(object_ownership) definition. - - Management of bucket ownership controls requires botocore>=1.18.11. default: false type: bool version_added: 2.0.0 @@ -847,8 +845,6 @@ def get_bucket_ownership_cntrl(s3_client, module, bucket_name): ''' Get current bucket public access block ''' - if not module.botocore_at_least('1.18.11'): - return None try: bucket_ownership = s3_client.get_bucket_ownership_controls(Bucket=bucket_name) return bucket_ownership['OwnershipControls']['Rules'][0]['ObjectOwnership'] @@ -1028,9 +1024,6 @@ def main(): delete_object_ownership = module.params.get('delete_object_ownership') object_ownership = module.params.get('object_ownership') - if delete_object_ownership or object_ownership: - module.require_botocore_at_least('1.18.11', reason='to manipulate bucket ownership controls') - # Parameter validation if encryption_key_id is not None and encryption != 'aws:kms': module.fail_json(msg="Only 'aws:kms' is a valid option for encryption parameter when you specify encryption_key_id.") diff --git a/requirements.txt b/requirements.txt index 3685e404330..1ff3ca2feb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # - tests/unit/constraints.txt # - tests/integration/constraints.txt # - tests/integration/targets/setup_botocore_pip -botocore>=1.18.0 -boto3>=1.15.0 +botocore>=1.19.0 +boto3>=1.16.0 # Final released version boto>=2.49.0 diff --git a/tests/integration/constraints.txt b/tests/integration/constraints.txt index bd95eb26733..e14f957c9c8 100644 --- a/tests/integration/constraints.txt +++ b/tests/integration/constraints.txt @@ -1,7 +1,7 @@ # Specifically run tests against the oldest versions that we support -boto3==1.15.0 -botocore==1.18.0 +boto3==1.16.0 +botocore==1.19.0 # AWS CLI has `botocore==` dependencies, provide the one that matches botocore # to avoid needing to download over a years worth of awscli wheels. -awscli==1.18.141 +awscli==1.18.160 diff --git a/tests/integration/targets/s3_bucket/roles/s3_bucket/tasks/ownership_controls.yml b/tests/integration/targets/s3_bucket/roles/s3_bucket/tasks/ownership_controls.yml index 233782dc4ef..209b658241b 100644 --- a/tests/integration/targets/s3_bucket/roles/s3_bucket/tasks/ownership_controls.yml +++ b/tests/integration/targets/s3_bucket/roles/s3_bucket/tasks/ownership_controls.yml @@ -6,110 +6,100 @@ security_token: "{{ security_token | default(omit) }}" region: "{{ aws_region }}" block: - - include_role: - name: setup_botocore_pip - vars: - botocore_version: '1.18.11' - - # ============================================================ - - name: Wrap test in virtualenv - vars: - ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" - block: - - set_fact: - local_bucket_name: "{{ bucket_name | hash('md5')}}ownership" - - - name: 'Create a simple bucket bad value for ownership controls' - s3_bucket: - name: '{{ local_bucket_name }}' - state: present - object_ownership: default - ignore_errors: true - register: output - - - assert: - that: - - output.failed - - - name: 'Create bucket with object_ownership set to object_writer' - s3_bucket: - name: '{{ local_bucket_name }}' - state: present - ignore_errors: true - register: output - - - assert: - that: - - output.changed - - not output.object_ownership|bool - - - name: delete s3 bucket - s3_bucket: - name: '{{ local_bucket_name }}' - state: absent - - - name: 'create s3 bucket with object ownership controls' - s3_bucket: - name: '{{ local_bucket_name }}' - state: present - object_ownership: ObjectWriter - register: output - - - assert: - that: - - output.changed - - output.object_ownership - - output.object_ownership == 'ObjectWriter' - - - name: 'update s3 bucket ownership controls' - s3_bucket: - name: '{{ local_bucket_name }}' - state: present - object_ownership: BucketOwnerPreferred - register: output - - - assert: - that: - - output.changed - - output.object_ownership - - output.object_ownership == 'BucketOwnerPreferred' - - - name: 'test idempotency update s3 bucket ownership controls' - s3_bucket: - name: '{{ local_bucket_name }}' - state: present - object_ownership: BucketOwnerPreferred - register: output - - - assert: - that: - - output.changed is false - - output.object_ownership - - output.object_ownership == 'BucketOwnerPreferred' - - - name: 'delete s3 bucket ownership controls' - s3_bucket: - name: '{{ local_bucket_name }}' - state: present - delete_object_ownership: true - register: output - - - assert: - that: - - output.changed - - not output.object_ownership|bool - - - name: 'delete s3 bucket ownership controls once again (idempotency)' - s3_bucket: - name: '{{ local_bucket_name }}' - state: present - delete_object_ownership: true - register: idempotency - - - assert: - that: - - not idempotency.changed - - not idempotency.object_ownership|bool + - set_fact: + local_bucket_name: "{{ bucket_name | hash('md5')}}ownership" + + - name: 'Create a simple bucket bad value for ownership controls' + s3_bucket: + name: '{{ local_bucket_name }}' + state: present + object_ownership: default + ignore_errors: true + register: output + + - assert: + that: + - output.failed + + - name: 'Create bucket with object_ownership set to object_writer' + s3_bucket: + name: '{{ local_bucket_name }}' + state: present + ignore_errors: true + register: output + + - assert: + that: + - output.changed + - not output.object_ownership|bool + + - name: delete s3 bucket + s3_bucket: + name: '{{ local_bucket_name }}' + state: absent + + - name: 'create s3 bucket with object ownership controls' + s3_bucket: + name: '{{ local_bucket_name }}' + state: present + object_ownership: ObjectWriter + register: output + + - assert: + that: + - output.changed + - output.object_ownership + - output.object_ownership == 'ObjectWriter' + + - name: 'update s3 bucket ownership controls' + s3_bucket: + name: '{{ local_bucket_name }}' + state: present + object_ownership: BucketOwnerPreferred + register: output + + - assert: + that: + - output.changed + - output.object_ownership + - output.object_ownership == 'BucketOwnerPreferred' + + - name: 'test idempotency update s3 bucket ownership controls' + s3_bucket: + name: '{{ local_bucket_name }}' + state: present + object_ownership: BucketOwnerPreferred + register: output + + - assert: + that: + - output.changed is false + - output.object_ownership + - output.object_ownership == 'BucketOwnerPreferred' + + - name: 'delete s3 bucket ownership controls' + s3_bucket: + name: '{{ local_bucket_name }}' + state: present + delete_object_ownership: true + register: output + + - assert: + that: + - output.changed + - not output.object_ownership|bool + + - name: 'delete s3 bucket ownership controls once again (idempotency)' + s3_bucket: + name: '{{ local_bucket_name }}' + state: present + delete_object_ownership: true + register: idempotency + + - assert: + that: + - not idempotency.changed + - not idempotency.object_ownership|bool # ============================================================ always: diff --git a/tests/integration/targets/setup_botocore_pip/defaults/main.yml b/tests/integration/targets/setup_botocore_pip/defaults/main.yml index 5a50b775907..2f3fefff35d 100644 --- a/tests/integration/targets/setup_botocore_pip/defaults/main.yml +++ b/tests/integration/targets/setup_botocore_pip/defaults/main.yml @@ -1,2 +1,2 @@ -default_botocore_version: '1.18.0' -default_boto3_version: '1.15.0' +default_botocore_version: '1.19.0' +default_boto3_version: '1.16.0' diff --git a/tests/unit/constraints.txt b/tests/unit/constraints.txt index bd95eb26733..e14f957c9c8 100644 --- a/tests/unit/constraints.txt +++ b/tests/unit/constraints.txt @@ -1,7 +1,7 @@ # Specifically run tests against the oldest versions that we support -boto3==1.15.0 -botocore==1.18.0 +boto3==1.16.0 +botocore==1.19.0 # AWS CLI has `botocore==` dependencies, provide the one that matches botocore # to avoid needing to download over a years worth of awscli wheels. -awscli==1.18.141 +awscli==1.18.160 diff --git a/tests/unit/module_utils/core/ansible_aws_module/test_minimal_versions.py b/tests/unit/module_utils/core/ansible_aws_module/test_minimal_versions.py index 911b05726f8..92373392271 100644 --- a/tests/unit/module_utils/core/ansible_aws_module/test_minimal_versions.py +++ b/tests/unit/module_utils/core/ansible_aws_module/test_minimal_versions.py @@ -20,10 +20,10 @@ class TestMinimalVersions(object): # Prepare some data for use in our testing # ======================================================== def setup_method(self): - self.MINIMAL_BOTO3 = '1.15.0' - self.MINIMAL_BOTOCORE = '1.18.0' - self.OLD_BOTO3 = '1.14.999' - self.OLD_BOTOCORE = '1.17.999' + self.MINIMAL_BOTO3 = '1.16.0' + self.MINIMAL_BOTOCORE = '1.19.0' + self.OLD_BOTO3 = '1.15.999' + self.OLD_BOTOCORE = '1.18.999' # ======================================================== # Test we don't warn when using valid versions @@ -142,8 +142,8 @@ def test_warn_botocore(self, monkeypatch, stdin, capfd): # ======================================================== @pytest.mark.parametrize("stdin", [{}], indirect=["stdin"]) def test_warn_boto3_and_botocore(self, monkeypatch, stdin, capfd): - monkeypatch.setattr(botocore, "__version__", "1.15.999") - monkeypatch.setattr(boto3, "__version__", "1.12.999") + monkeypatch.setattr(botocore, "__version__", self.OLD_BOTOCORE) + monkeypatch.setattr(boto3, "__version__", self.OLD_BOTO3) # Create a minimal module that we can call module = AnsibleAWSModule(argument_spec=dict()) From c45ab3c8a73d3ec9e681c42b10eec2ff2c23f317 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Mon, 22 Nov 2021 18:10:43 +0100 Subject: [PATCH 2/2] changelog --- changelogs/fragments/574-botocore-1-19-0.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelogs/fragments/574-botocore-1-19-0.yml diff --git a/changelogs/fragments/574-botocore-1-19-0.yml b/changelogs/fragments/574-botocore-1-19-0.yml new file mode 100644 index 00000000000..13324282d4f --- /dev/null +++ b/changelogs/fragments/574-botocore-1-19-0.yml @@ -0,0 +1,7 @@ +major_changes: +- amazon.aws collection - The amazon.aws collection has dropped support for + ``botocore<1.19.0`` and ``boto3<1.16.0``. Most modules will continue to work + with older versions of the AWS SDK, however compatability with older versions + of the SDK is not guaranteed and will not be tested. When using older versions + of the SDK a warning will be emitted by Ansible + (https://github.com/ansible-collections/amazon.aws/pull/574).