Deploy off aws cli (#1455)

* feat: sam deploy without aws cli pre-installed

- Not breaking parameter overrides formats
- still requires refactoring and error handling

* feat: new click types for deploy parameters

* feat: show changeset and stack events

- needs refactoring

* refactor: move deploy classes to lib

- wire up command.py for `sam deploy`
- move deploy specific exceptions to inherit from UserException

* rebase: latest from `sam package` port

* feat: decorator for printing tables

- `sam deploy` now has tables while showcasing the changeset and
  showcasing events happening during deploy.

* fix: wrap text on resource status column on `sam deploy`

- fixed unit tests
- linting fixes
- doc strings
- further unit tests and integration tests need to be added.

* fix: cleaner text formatting for tables

* tests: add unit tests for full suite of `sam deploy`

* tests: add integration tests for `sam deploy`

* tests: regression test suite for `sam deploy`

- exercise all command line parameters for `aws` and `sam`

* fix: deploy command now showcases stack outputs

* fix: address comments

* fix: return stack outputs from `get_stack_outputs`

* fix: width margins on table prints

* fix: address comments

- add retries
- more regression testing
- remove types for capabilities

* tests: tests for pprint of tables

* usability: add table headers

- show cases Add, Modify, Delete with +, * and -

.pylintrc

@@ -59,7 +59,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=R0201,W0613,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904,C0301
+disable=R0201,W0613,W0640,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904,C0301
 
 
 [REPORTS]

samcli/cli/types.py

@@ -15,27 +15,51 @@ class CfnParameterOverridesType(click.ParamType):
     parameters as "ParameterKey=KeyPairName,ParameterValue=MyKey ParameterKey=InstanceType,ParameterValue=t1.micro"
     """
 
-    __EXAMPLE = "ParameterKey=KeyPairName,ParameterValue=MyKey ParameterKey=InstanceType,ParameterValue=t1.micro"
+    __EXAMPLE_1 = "ParameterKey=KeyPairName,ParameterValue=MyKey ParameterKey=InstanceType,ParameterValue=t1.micro"
+    __EXAMPLE_2 = "KeyPairName=MyKey InstanceType=t1.micro"
 
     # Regex that parses CloudFormation parameter key-value pairs: https://regex101.com/r/xqfSjW/2
-    _pattern = r"(?:ParameterKey=([A-Za-z0-9\"]+),ParameterValue=(\"(?:\\.|[^\"\\]+)*\"|(?:\\.|[^ \"\\]+)+))"
+    _pattern_1 = r"(?:ParameterKey=([A-Za-z0-9\"]+),ParameterValue=(\"(?:\\.|[^\"\\]+)*\"|(?:\\.|[^ \"\\]+)+))"
+    _pattern_2 = r"(?:([A-Za-z0-9\"]+)=(\"(?:\\.|[^\"\\]+)*\"|(?:\\.|[^ \"\\]+)+))"
 
+    ordered_pattern_match = [_pattern_1, _pattern_2]
+
+    # NOTE(TheSriram): name needs to be added to click.ParamType requires it.
     name = ""
 
     def convert(self, value, param, ctx):
         result = {}
-        if not value:
-            return result
 
-        groups = re.findall(self._pattern, value)
-        if not groups:
-            return self.fail(
-                "{} is not in valid format. It must look something like '{}'".format(value, self.__EXAMPLE), param, ctx
-            )
+        # Empty tuple
+        if value == ("",):
+            return result
 
-        # 'groups' variable is a list of tuples ex: [(key1, value1), (key2, value2)]
-        for key, param_value in groups:
-            result[self._unquote(key)] = self._unquote(param_value)
+        for val in value:
+
+            try:
+                # NOTE(TheSriram): find the first regex that matched.
+                # pylint is concerned that we are checking at the same `val` within the loop,
+                # but that is the point, so disabling it.
+                pattern = next(
+                    i
+                    for i in filter(
+                        lambda item: re.findall(item, val), self.ordered_pattern_match
+                    )  # pylint: disable=cell-var-from-loop
+                )
+            except StopIteration:
+                return self.fail(
+                    "{} is not in valid format. It must look something like '{}' or '{}'".format(
+                        val, self.__EXAMPLE_1, self.__EXAMPLE_2
+                    ),
+                    param,
+                    ctx,
+                )
+
+            groups = re.findall(pattern, val)
+
+            # 'groups' variable is a list of tuples ex: [(key1, value1), (key2, value2)]
+            for key, param_value in groups:
+                result[self._unquote(key)] = self._unquote(param_value)
 
         return result
 
@@ -80,7 +104,7 @@ class CfnMetadataType(click.ParamType):
     _pattern = r"([A-Za-z0-9\"]+)=([A-Za-z0-9\"]+)"
 
     # NOTE(TheSriram): name needs to be added to click.ParamType requires it.
-    name = "CfnMetadata"
+    name = ""
 
     def convert(self, value, param, ctx):
         result = {}
@@ -103,13 +127,54 @@ def convert(self, value, param, ctx):
             if not groups:
                 fail = True
             for group in groups:
-                key, value = group
+                key, v = group
                 # assign to result['KeyName1'] = string and so on.
-                result[key] = value
+                result[key] = v
 
         if fail:
             return self.fail(
                 "{} is not in valid format. It must look something like '{}'".format(value, self._EXAMPLE), param, ctx
             )
 
         return result
+
+
+class CfnTags(click.ParamType):
+    """
+    Custom Click options type to accept values for tag parameters.
+    tag parameters can be of the type KeyName1=string KeyName2=string
+    """
+
+    _EXAMPLE = "KeyName1=string KeyName2=string"
+
+    _pattern = r"([A-Za-z0-9\"]+)=([A-Za-z0-9\"]+)"
+
+    # NOTE(TheSriram): name needs to be added to click.ParamType requires it.
+    name = ""
+
+    def convert(self, value, param, ctx):
+        result = {}
+        fail = False
+        # Empty tuple
+        if value == ("",):
+            return result
+
+        for val in value:
+
+            groups = re.findall(self._pattern, val)
+
+            if not groups:
+                fail = True
+            for group in groups:
+                key, v = group
+                # assign to result['KeyName1'] = string and so on.
+                result[key] = v
+
+            if fail:
+                return self.fail(
+                    "{} is not in valid format. It must look something like '{}'".format(value, self._EXAMPLE),
+                    param,
+                    ctx,
+                )
+
+        return result

samcli/commands/_utils/custom_options/option_nargs.py

@@ -0,0 +1,50 @@
+"""
+Custom Click options for multiple arguments
+"""
+
+import click
+
+
+class OptionNargs(click.Option):
+    """
+    A custom option class that allows parsing for multiple arguments
+    for an option, when the number of arguments for an option are unknown.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.nargs = kwargs.pop("nargs", -1)
+        super(OptionNargs, self).__init__(*args, **kwargs)
+        self._previous_parser_process = None
+        self._nargs_parser = None
+
+    def add_to_parser(self, parser, ctx):
+        def parser_process(value, state):
+            # look ahead into arguments till we reach the next option.
+            # the next option starts with a prefix which is either '-' or '--'
+            next_option = False
+            value = [value]
+
+            while state.rargs and not next_option:
+                for prefix in self._nargs_parser.prefixes:
+                    if state.rargs[0].startswith(prefix):
+                        next_option = True
+                if not next_option:
+                    value.append(state.rargs.pop(0))
+
+            value = tuple(value)
+
+            # call the actual process
+            self._previous_parser_process(value, state)
+
+        # Add current option to Parser by calling add_to_parser on the super class.
+        super(OptionNargs, self).add_to_parser(parser, ctx)
+        for name in self.opts:
+            # Get OptionParser object for current option
+            option_parser = getattr(parser, "_long_opt").get(name) or getattr(parser, "_short_opt").get(name)
+            if option_parser:
+                # Monkey patch `process` method for click.parser.Option class.
+                # This allows for setting multiple parsed values into current option arguments
+                self._nargs_parser = option_parser
+                self._previous_parser_process = option_parser.process
+                option_parser.process = parser_process
+                break

samcli/commands/_utils/options.py

@@ -7,7 +7,9 @@
 from functools import partial
 
 import click
-from samcli.cli.types import CfnParameterOverridesType, CfnMetadataType
+from samcli.cli.types import CfnParameterOverridesType, CfnMetadataType, CfnTags
+from samcli.commands._utils.custom_options.option_nargs import OptionNargs
+
 
 _TEMPLATE_OPTION_DEFAULT_VALUE = "template.[yaml|yml]"
 
@@ -113,6 +115,7 @@ def docker_click_options():
 def parameter_override_click_option():
     return click.option(
         "--parameter-overrides",
+        cls=OptionNargs,
         type=CfnParameterOverridesType(),
         help="Optional. A string that contains CloudFormation parameter overrides encoded as key=value "
         "pairs. Use the same format as the AWS CLI, e.g. 'ParameterKey=KeyPairName,"
@@ -134,3 +137,58 @@ def metadata_click_option():
 
 def metadata_override_option(f):
     return metadata_click_option()(f)
+
+
+def capabilities_click_option():
+    return click.option(
+        "--capabilities",
+        cls=OptionNargs,
+        type=click.STRING,
+        required=True,
+        help="A list of  capabilities  that  you  must  specify"
+        "before  AWS  Cloudformation  can create certain stacks. Some stack tem-"
+        "plates might include resources that can affect permissions in your  AWS"
+        "account,  for  example, by creating new AWS Identity and Access Manage-"
+        "ment (IAM) users. For those stacks,  you  must  explicitly  acknowledge"
+        "their  capabilities by specifying this parameter. The only valid values"
+        "are CAPABILITY_IAM and CAPABILITY_NAMED_IAM. If you have IAM resources,"
+        "you  can specify either capability. If you have IAM resources with cus-"
+        "tom names, you must specify CAPABILITY_NAMED_IAM. If you don't  specify"
+        "this  parameter, this action returns an InsufficientCapabilities error.",
+    )
+
+
+def capabilities_override_option(f):
+    return capabilities_click_option()(f)
+
+
+def tags_click_option():
+    return click.option(
+        "--tags",
+        cls=OptionNargs,
+        type=CfnTags(),
+        required=False,
+        help="A list of tags to associate with the stack that is created or updated."
+        "AWS CloudFormation also propagates these tags to resources "
+        "in the stack if the resource supports it.",
+    )
+
+
+def tags_override_option(f):
+    return tags_click_option()(f)
+
+
+def notification_arns_click_option():
+    return click.option(
+        "--notification-arns",
+        cls=OptionNargs,
+        type=click.STRING,
+        required=False,
+        help="Amazon  Simple  Notification  Service  topic"
+        "Amazon  Resource  Names  (ARNs) that AWS CloudFormation associates with"
+        "the stack.",
+    )
+
+
+def notification_arns_override_option(f):
+    return notification_arns_click_option()(f)

samcli/commands/_utils/table_print.py

@@ -0,0 +1,110 @@
+"""
+Utilities for table pretty printing using click
+"""
+from itertools import count, zip_longest
+import textwrap
+from functools import wraps
+
+import click
+
+
+def pprint_column_names(format_string, format_kwargs, margin=None, table_header=None):
+    """
+
+    :param format_string: format string to be used that has the strings, minimum width to be replaced
+    :param format_kwargs: dictionary that is supplied to the format_string to format the string
+    :param margin: margin that is to be reduced from column width for columnar text.
+    :param table_header: Supplied table header
+    :return: boilerplate table string
+    """
+
+    min_width = 100
+    min_margin = 2
+
+    def pprint_wrap(func):
+        # Calculate terminal width, number of columns in the table
+        width, _ = click.get_terminal_size()
+        # For UX purposes, set a minimum width for the table to be usable
+        # and usable_width keeps margins in mind.
+        width = max(width, min_width)
+
+        total_args = len(format_kwargs)
+        if not total_args:
+            raise ValueError("Number of arguments supplied should be > 0 , format_kwargs: {}".format(format_kwargs))
+
+        # Get width to be a usable number so that we can equally divide the space for all the columns.
+        # Can be refactored, to allow for modularity in the shaping of the columns.
+        width = width - (width % total_args)
+        usable_width_no_margin = int(width) - 1
+        usable_width = int((usable_width_no_margin - (margin if margin else min_margin)))
+        if total_args > int(usable_width / 2):
+            raise ValueError("Total number of columns exceed available width")
+        width_per_column = int(usable_width / total_args)
+
+        # The final column should not roll over into the next line
+        final_arg_width = width_per_column - 1
+
+        # the format string contains minimumwidth that need to be set.
+        # eg: "{a:{0}}} {b:<{1}}} {c:{2}}}"
+        format_args = [width_per_column for _ in range(total_args - 1)]
+        format_args.extend([final_arg_width])
+
+        # format arguments are now ready for setting minimumwidth
+
+        @wraps(func)
+        def wrap(*args, **kwargs):
+            # The table is setup with the column names, format_string contains the column names.
+            if table_header:
+                click.secho("\n" + table_header)
+            click.secho("-" * usable_width)
+            click.secho(format_string.format(*format_args, **format_kwargs))
+            click.secho("-" * usable_width)
+            # format_args which have the minimumwidth set per {} in the format_string is passed to the function
+            # which this decorator wraps, so that the function has access to the correct format_args
+            kwargs["format_args"] = format_args
+            kwargs["width"] = width_per_column
+            kwargs["margin"] = margin if margin else min_margin
+            result = func(*args, **kwargs)
+            # Complete the table
+            click.secho("-" * usable_width)
+            return result
+
+        return wrap
+
+    return pprint_wrap
+
+
+def wrapped_text_generator(texts, width, margin):
+    """
+
+    Return a generator where the contents are wrapped text to a specified width.
+
+    :param texts: list of text that needs to be wrapped at specified width
+    :param width: width of the text to be wrapped
+    :param margin: margin to be reduced from width for cleaner UX
+    :return: generator of wrapped text
+    """
+    for text in texts:
+        yield textwrap.wrap(text, width=width - margin)
+
+
+def pprint_columns(columns, width, margin, format_string, format_args, columns_dict):
+    """
+
+    Print columns based on list of columnar text, associated formatting string and associated format arguments.
+
+    :param columns: list of columnnar text that go into columns as specified by the format_string
+    :param width: width of the text to be wrapped
+    :param margin: margin to be reduced from width for cleaner UX
+    :param format_string: A format string that has both width and text specifiers set.
+    :param format_args: list of offset specifiers
+    :param columns_dict: arguments dictionary that have dummy values per column
+    :return:
+    """
+    for columns_text in zip_longest(*wrapped_text_generator(columns, width, margin), fillvalue=""):
+        counter = count()
+        # Generate columnar data that correspond to the column names and update them.
+        for k, _ in columns_dict.items():
+            columns_dict[k] = columns_text[next(counter)]
+
+        click.secho(format_string.format(*format_args, **columns_dict))