Merge pull request #524 from cloudify-cosmo/3.1.7-build

support client config from plugin yaml

CHANGELOG.txt

@@ -1,3 +1,4 @@
+3.1.6: add __version__.py file in cloudify_aws folder.
 3.1.5: RD-6735 Do not stop existing resource.
 3.1.4: Update Check Drift Status
 3.1.3: Add account ID property to roles in precreate.

cloudify_aws/__version__.py

@@ -0,0 +1 @@
+version = '3.1.6'

cloudify_aws/common/__init__.py

@@ -49,7 +49,7 @@ def __init__(self, client, resource_id=None, logger=None):
         set_stream_logger('botocore.parsers', level=DEBUG)
         self.logger = logger or init_cloudify_logger(NullHandler(),
                                                      'AWSResourceBase')
-        self.client = client
+        self._client = client
         self._resource_id = text_type(resource_id) if resource_id else None
         self._initial_configuration = None  # resource_config from node props
         self._create_response = None  # create_response
@@ -58,6 +58,14 @@ def __init__(self, client, resource_id=None, logger=None):
         self._previous_configuration = None  # Previous extrapolation
         self._describe_call = None
 
+    @property
+    def client(self):
+        return self._client
+
+    @client.setter
+    def client(self, value):
+        self._client = value
+
     def get_describe_result(self, params):
         try:
             return self.make_client_call(self._describe_call, params)

cloudify_aws/common/connection.py

@@ -27,8 +27,7 @@
     get_uuid,
     desecretize_client_config
 )
-from cloudify import ctx
-from cloudify_aws.common.constants import AWS_CONFIG_PROPERTY
+from cloudify_common_sdk.utils import get_client_config
 
 # pylint: disable=R0903
 
@@ -50,44 +49,48 @@ def __init__(self, node, aws_config=None):
             'aws_session_token',
             'api_version']
 
-        config_from_props = node.properties.get(AWS_CONFIG_PROPERTY, dict())
+        config_from_utils = get_client_config(
+            ctx_node=node, alternate_key='aws_config')
+
+        # config_from_props = node.properties.get(AWS_CONFIG_PROPERTY, dict())
         # Get additional config from node configuration.
-        additional_config = config_from_props.pop('additional_config', None)
+        additional_config = config_from_utils.pop('additional_config', None)
 
         # Handle the Plugin properties
-        config_from_plugin_props = getattr(ctx.plugin, 'properties', {})
-        additional_config_plugin = config_from_plugin_props.pop(
-            'additional_config', None)
+        # config_from_plugin_props = getattr(ctx.plugin, 'properties', {})
+        # additional_config_plugin = config_from_plugin_props.pop(
+        #     'additional_config', None)
 
-        config_from_plugin_props.update(config_from_props)
-        self.aws_config = desecretize_client_config(
-            config_from_plugin_props)
+        # config_from_plugin_props.update(config_from_props)
         # Merge user-provided AWS config with generated config
+        self._aws_config = desecretize_client_config(
+            config_from_utils)
+        self._aws_config['region_name'] = self._aws_config.get('region_name')
         if aws_config:
-            self.aws_config.update(aws_config)
+            self._aws_config.update(aws_config)
 
         # Prepare region name for Boto
-        self.aws_config['region_name'] = self.aws_config.get('region_name')
 
         # This it check if "aws_config" contains "endpoint_url" or not
         for option in aws_config_options:
-            if self.aws_config.get(option):
+            if self._aws_config.get(option):
                 aws_config_whitelist.append(option)
 
         # Delete all non-whitelisted keys
-        self.aws_config = {k: v for k, v in self.aws_config.items()
-                           if k in aws_config_whitelist}
+        self._aws_config = {k: v for k, v in self.aws_config.items()
+                            if k in aws_config_whitelist}
 
         # Add additional config after whitelist filter.
         if additional_config and isinstance(additional_config, dict):
-            self.aws_config['config'] = Config(**additional_config)
-        if additional_config_plugin and isinstance(
-                additional_config_plugin, dict):
-            self.aws_config['config'].update(additional_config_plugin)
-        else:
-            ctx.logger.debug(
-                'No plugin properties were provided. '
-                'Defaulting client_config credentials.')
+            self._aws_config['config'] = Config(**additional_config)
+
+    @property
+    def aws_config(self):
+        return self._aws_config
+
+    @aws_config.setter
+    def aws_config(self, value):
+        self._aws_config = value
 
     def get_sts_client(self, config):
         return boto3.client("sts", **config)
@@ -120,8 +123,28 @@ def client(self, service_name):
         :returns: An AWS service Boto3 client
         :raises: :exc:`cloudify.exceptions.NonRecoverableError`
         '''
-        config = self.aws_config
-        assume_role = self.aws_config.pop('assume_role', None) \
+        config = self._aws_config
+        assume_role = self._aws_config.pop('assume_role', None) \
+            or os.environ.get("AWS_ASSUME_ROLE_ARN")
+
+        if assume_role:
+            config = self.get_sts_credentials(assume_role, config)
+
+        resource = boto3.client(service_name, **config)
+        return resource
+
+    def client_with_region(self, service_name, region_name):
+        '''
+            Builds an AWS connection client
+
+        :param str service_name: A Boto3 service name
+        :returns: An AWS service Boto3 client
+        :raises: :exc:`cloudify.exceptions.NonRecoverableError`
+        '''
+        config = self._aws_config
+        config['region_name'] = region_name
+
+        assume_role = self._aws_config.pop('assume_role', None) \
             or os.environ.get("AWS_ASSUME_ROLE_ARN")
 
         if assume_role:

cloudify_aws/common/decorators.py

@@ -32,8 +32,8 @@
 from cloudify_common_sdk.utils import \
     skip_creative_or_destructive_operation as skip
 from cloudify_common_sdk.utils import (
+    v1_gteq_v2,
     get_cloudify_version,
-    v1_gteq_v2
 )
 # Local imports
 from .constants import SUPPORT_DRIFT

cloudify_aws/ec2/__init__.py

@@ -19,9 +19,9 @@
 
 # Cloudify AWS
 from cloudify_aws.common import AWSResourceBase
-from cloudify_aws.common.connection import Boto3Connection
-from cloudify_aws.common.constants import AWS_CONFIG_PROPERTY
 from cloudify_aws.common.utils import check_region_name
+from cloudify_common_sdk.utils import get_client_config
+from cloudify_aws.common.connection import Boto3Connection
 
 # pylint: disable=R0903
 
@@ -37,8 +37,9 @@ def __init__(self,
                  logger=None):
 
         if not client:
-            aws_config = ctx_node.properties.get(AWS_CONFIG_PROPERTY, dict())
-            check_region_name(aws_config['region_name'])
+            config_from_utils = get_client_config(
+               ctx_node=ctx_node, alternate_key='aws_config')
+            check_region_name(config_from_utils.get('region_name'))
         AWSResourceBase.__init__(
             self, client or Boto3Connection(ctx_node).client('ec2'),
             resource_id=resource_id, logger=logger)

cloudify_aws/ec2/resources/image.py

@@ -17,9 +17,10 @@
     AWS EC2 Image interface
 """
 # Cloudify
-from cloudify.exceptions import NonRecoverableError
-from cloudify_aws.common import decorators, utils
 from cloudify_aws.ec2 import EC2Base
+from cloudify_aws.common import decorators, utils
+from cloudify.exceptions import NonRecoverableError
+
 # Boto
 from botocore.exceptions import ClientError, ParamValidationError
 
@@ -56,13 +57,10 @@ def __init__(self, ctx_node, resource_id=None, client=None, logger=None):
     def properties(self):
         """Gets the properties of an external resource"""
         params = self.describe_image_filters
-        self.logger.info('Params: {}'.format(params))
         if not params:
             return
         try:
-            resources = \
-                self.client.describe_images(**params)
-            self.logger.info('Describe images returned: {}'.format(resources))
+            resources = self.client.describe_images(**params)
         except (ClientError, ParamValidationError):
             pass
         else:

cloudify_aws/ec2/resources/subnet.py

@@ -20,14 +20,17 @@
 from deepdiff import DeepDiff
 
 # Boto
-from botocore.exceptions import ClientError
-from botocore.exceptions import CapacityNotAvailableError
+from botocore.exceptions import (
+    ClientError,
+    CapacityNotAvailableError
+)
 
 # Cloudify
 from cloudify.exceptions import NonRecoverableError, OperationRetry
 
 from cloudify_aws.ec2 import EC2Base
 from cloudify_aws.common import decorators, utils
+from cloudify_common_sdk.utils import get_client_config
 from cloudify_aws.common.constants import EXTERNAL_RESOURCE_ID
 
 RESOURCE_TYPE = 'EC2 Subnet'
@@ -208,11 +211,17 @@ def poststart(ctx, iface=None, **_):
 
 def _create(ctx_node, iface, params, logger):
     # Actually create the resource
-    region_name = ctx_node.properties['client_config']['region_name']
-    use_available_zones = ctx_node.properties.get('use_available_zones', False)
     try:
         iface.create(params)
-    except CapacityNotAvailableError:
+    except (NonRecoverableError, CapacityNotAvailableError) as e:
+        if isinstance(e, NonRecoverableError) and \
+                'InvalidParameterValue' not in str(e):
+            raise e
+        config_from_utils = get_client_config(
+            ctx_node=ctx_node, alternate_key='aws_config')
+        region_name = config_from_utils.get('region_name')
+        use_available_zones = ctx_node.properties.get(
+            'use_available_zones', False)
         if use_available_zones:
             logger.error(
                 "The Availability Zone chosen {0} "

cloudify_aws/ec2/tests/test_ec2.py

@@ -21,14 +21,16 @@
 from cloudify.exceptions import NonRecoverableError
 
 # Local imports
-from cloudify_aws.common.tests.test_base import TestServiceBase
 from cloudify_aws.ec2 import EC2Base
 from cloudify_aws.common.constants import AWS_CONFIG_PROPERTY
+from cloudify_aws.common.tests.test_base import TestServiceBase
 
 
 class TestEC2Init(TestServiceBase):
 
-    def test_credentials(self):
+    @mock.patch('cloudify_common_sdk.utils.get_ctx_instance')
+    @mock.patch('cloudify_common_sdk.utils.get_ctx_plugin')
+    def test_credentials(self, mock_plugin_ctx, *_):
         boto_client = mock.Mock()
         boto_mock = mock.Mock(return_value=boto_client)
         ctx_node = mock.Mock()
@@ -37,6 +39,9 @@ def test_credentials(self):
                 'region_name': 'wr-ongvalu-e'
             }
         }
+        mock_plugin_ctx.return_value = {
+            'foo': 'bar'
+        }
         with mock.patch(
             "cloudify_aws.ec2.Boto3Connection", boto_mock
         ):