Merge pull request #540 from cloudify-cosmo/3.1.7-build

3.1.7 build

.circleci/config.yml

@@ -41,6 +41,28 @@ executors:
 
 commands:
 
+  ec2_image_verify:
+    steps:
+      - run: |
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+      - run: |
+          aws configure set default.region us-east-1
+          aws configure set aws_access_key_id $(echo $aws_access_key_id | base64 -d -w 0)
+          aws configure set aws_secret_access_key $(echo $aws_secret_access_key | base64 -d -w 0)
+          aws configure set output json
+      - run: |
+          ret=`aws ec2 describe-images --filters Name=name,Values=CentOS7-cloudify-examples-image | jq '.Images' | jq -r '.[0].Name'`
+          if [ $ret != "CentOS7-cloudify-examples-image" ]; then
+              echo "The current AWS account cannot find the required examples image. Tests cannot pass.";
+              exit 1;
+          else
+              echo "Found it!";
+              exit 0;
+          fi
+
+
   eks_prepare_test_manager:
     steps:
       - run: ls -alR
@@ -66,7 +88,7 @@ commands:
 
   run_cf_test:
     steps:
-      - run: ecosystem-test local-blueprint-test -b examples/blueprint-examples/virtual-machine/aws-cloudformation.yaml --test-id=virtual-machine -i aws_region_name=eu-west-3 -i resource_suffix=$CIRCLE_BUILD_NUM --on-failure=uninstall-force --timeout=3000
+      - run: ecosystem-test local-blueprint-test -b examples/virtual-machine/aws-cloudformation.yaml --test-id=virtual-machine -i aws_region_name=eu-west-3 -i resource_suffix=$CIRCLE_BUILD_NUM --on-failure=uninstall-force --timeout=3000
 
   run_eks_test:
     steps:
@@ -138,12 +160,11 @@ commands:
 
 jobs:
 
-  run_sample_job:
-    executor: cloudify-machine-py3
-    environment:
-      CLOUDIFY_SSL_TRUST_ALL: true
+  verify_ec2_image:
+    executor: py36
     steps:
-      - run_sample_test
+      - checkout
+      - ec2_image_verify
 
   cf_integration_tests_py3:
     executor: cloudify-machine-py3
@@ -353,6 +374,7 @@ workflows:
   version: 2
   tests:
     jobs:
+      - verify_ec2_image
       - node/check_py3_compat_job
       - node/unittests_job:
           context: 
@@ -430,6 +452,7 @@ workflows:
             only:
               - master
     jobs:
+      - verify_ec2_image
       - node/check_py3_compat_job
       - node/unittests_job:
           context: 
@@ -464,6 +487,7 @@ workflows:
             - slack-secrets
           <<: *job-post-steps
           requires:
+            - verify_ec2_image
             - wagonorb/wagon
             - wagonorb/rhel_wagon
             - wagonorb/arch64_wagon
@@ -475,6 +499,7 @@ workflows:
             - slack-secrets
           <<: *job-post-steps
           requires:
+            - verify_ec2_image
             - wagonorb/wagon
             - wagonorb/rhel_wagon
             - wagonorb/arch64_wagon
@@ -515,6 +540,7 @@ workflows:
             only:
               - master
     jobs:
+      - verify_ec2_image
       - node/check_py3_compat_job
       - node/unittests_job:
           context: 
@@ -549,6 +575,7 @@ workflows:
             - slack-secrets
           <<: *job-post-steps
           requires:
+            - verify_ec2_image
             - wagonorb/wagon
             - wagonorb/rhel_wagon
             - wagonorb/arch64_wagon
@@ -610,6 +637,7 @@ workflows:
             only:
               - master
     jobs:
+      - verify_ec2_image
       - node/check_py3_compat_job
       - node/unittests_job:
           context: 
@@ -633,6 +661,7 @@ workflows:
             - slack-secrets
           <<: *job-post-steps
           requires:
+            - verify_ec2_image
             - wagonorb/wagon
             - wagonorb/rhel_wagon
             - wagonorb/arch64_wagon
@@ -693,4 +722,5 @@ workflows:
             - wagonorb/arch64_wagon
           filters:
             branches:
-              only: /([0-9\.]*\-build|master|dev)/
+              only: /([0-9\.]*\-build|master|dev)/
+

CHANGELOG.txt

@@ -1,3 +1,7 @@
+3.1.7: 
+  - Fix bug after Cloudify 7 release.
+  - Support for ACL deprecation in s3 buckets.
+  - Fix bug in cloudformation-feature-demo blueprint
 3.1.6: add __version__.py file in cloudify_aws folder.
 3.1.5: RD-6735 Do not stop existing resource.
 3.1.4: Update Check Drift Status

cloudify_aws/__version__.py

@@ -1 +1 @@
-version = '3.1.6'
+version = '3.1.7'

cloudify_aws/s3/resources/bucket.py

@@ -17,8 +17,10 @@
     AWS S3 Bucket interface
 """
 # Cloudify
-from cloudify_aws.common import decorators
 from cloudify_aws.s3 import S3Base
+from cloudify_aws.common import decorators
+from cloudify.exceptions import NonRecoverableError
+
 # Boto
 from botocore.exceptions import ClientError, ParamValidationError
 
@@ -72,6 +74,20 @@ def delete(self, params=None):
                           % (self.type_name, params))
         self.client.delete_bucket(**params)
 
+    def put_public_access_block(self, params):
+        """
+            put PublicAccessBlock configuration.
+        """
+        self.client.put_public_access_block(
+            Bucket=params['Bucket'],
+            PublicAccessBlockConfiguration={
+                'BlockPublicAcls': True,
+                'IgnorePublicAcls': True,
+                'BlockPublicPolicy': False,
+                'RestrictPublicBuckets': False
+            }
+        )
+
     def delete_objects(self, bucket):
         list_objects = self.client.list_objects(Bucket=bucket)
         for object in list_objects.get('Contents', []):
@@ -109,7 +125,19 @@ def create(ctx, iface, resource_config, params, **_):
             del params['CreateBucketConfiguration']
 
     # Actually create the resource
-    bucket = iface.create(params)
+    try:
+        bucket = iface.create(params)
+    except NonRecoverableError as e:
+        acl = params.pop('ACL', '')
+        if "InvalidBucketAclWithObjectOwnership" not in str(e) \
+                and "Bucket cannot have ACLs" not in str(e) \
+                    and 'public-read' not in acl:
+            raise e
+        ctx.logger.error('Deprecation warning, the AWS API has changed \
+            and ACL-public is no longer valid.')
+        bucket = iface.create(params)
+
+    iface.put_public_access_block(params)
     ctx.instance.runtime_properties[LOCATION] = bucket.get(LOCATION)
 
 

cloudify_aws/s3/resources/bucket_object.py

@@ -247,7 +247,17 @@ def create(ctx, iface, resource_config, **_):
     ctx.instance.runtime_properties[BUCKET] = bucket_name
 
     # Actually create the resource
-    iface.create(resource_config)
+    try:
+        iface.create(resource_config)
+    except NonRecoverableError as e:
+        acl = resource_config.pop('ACL', '')
+        if 'AccessControlListNotSupported' not in str(e) \
+            and 'The bucket does not allow ACLs' not in str(e) \
+                and 'public-read' not in acl:
+            raise e
+        ctx.logger.error('Deprecation warning, the AWS API has changed and \
+                         ACL-public is no longer valid.')
+        iface.create(resource_config)
 
 
 @decorators.check_swift_resource

examples/cloudformation-feature-demo/blueprint.yaml

@@ -16,7 +16,7 @@ inputs:
 
   aws_region_name:
     type: string
-    default: { get_secret: ec2_region_name }
+    default: eu-west-2
 
   availability_zone:
     type: string
@@ -81,7 +81,6 @@ node_templates:
               HelloBucket:
                 Type: AWS::S3::Bucket
                 Properties:
-                  AccessControl: PublicRead
                   WebsiteConfiguration:
                     IndexDocument: index.html
                     ErrorDocument: error.html

examples/s3-feature-demo/blueprint.yaml

@@ -22,6 +22,9 @@ inputs:
     type: string
     default: { concat: [ { get_input: aws_region_name}, 'b' ] }
 
+  name_bucket:
+    type: string
+    default: cloudify-aws-testing-bucket
 
 dsl_definitions:
 
@@ -37,8 +40,7 @@ node_templates:
     type: cloudify.nodes.aws.s3.Bucket
     properties:
       resource_config:
-        Bucket: cloudify-aws-testing-bucket
-        ACL: public-read-write
+        Bucket: { get_input: name_bucket }
         CreateBucketConfiguration:
           LocationConstraint: { get_input: aws_region_name }
       client_config: *client_config
@@ -98,7 +100,6 @@ node_templates:
     properties:
       source_type: 'bytes'
       resource_config:
-        ACL: 'public-read'
         Bucket: { get_property: [ bucket, resource_config, Bucket ] }
         Key: 'test-byte-data.txt'
         kwargs:
@@ -116,7 +117,6 @@ node_templates:
       source_type: 'local'
       path: './local-s3-object.txt'
       resource_config:
-        ACL: 'public-read'
         Bucket: { get_property: [ bucket, resource_config, Bucket ] }
         Key: 'local-s3-object.txt'
       client_config: *client_config
@@ -132,7 +132,6 @@ node_templates:
       source_type: 'remote'
       path: 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf'
       resource_config:
-        ACL: 'public-read'
         Bucket: { get_property: [ bucket, resource_config, Bucket ] }
         Key: 'dummy.pdf'
       client_config: *client_config