Merge pull request #55 from MarketSquare/version_0.2.0

Version 0.2.0

.gitignore

@@ -132,5 +132,5 @@ dmypy.json
 
 #Pycharm
 .idea
-tests/robot/new_tests.robot
+tests/robot/local_tests.robot
 log/

CONTRIBUTING.md

@@ -1,5 +1,9 @@
 # ***** UNDER CONSTRUCTION *****
 
+We are working to create an environment with tests in localstack https://github.com/localstack/localstack if you have 
+experience with this tool and time, any help will be appreciated.
+
+
 ## Contributing to RobotFramework-AWS
 
 Thank you for considering contributing to a library for interacting with AWS Services in RobotFramework for Test Automation.

setup.py

@@ -9,7 +9,7 @@ def readme():
 
 setup(
     name='robotframework-aws',
-    version='0.1.0',
+    version='0.2.0',
     author="Dillan Teagle",
     author_email="[email protected]",
     description="A python package to test AWS services in Robot Framework",

src/AWSLibrary/__init__.py

@@ -4,7 +4,8 @@
     SessionKeywords,
     S3Keywords,
     ResourceKeywords,
-    DynamoKeywords
+    DynamoKeywords,
+    CloudWatchKeywords
 )
 from AWSLibrary.version import get_version
 __version__ = get_version()
@@ -45,5 +46,6 @@ def __init__(self):
             S3Keywords(self),
             ResourceKeywords(self),
             DynamoKeywords(self),
+            CloudWatchKeywords(self),
         ]
         DynamicCore.__init__(self, libraries)

src/AWSLibrary/keywords/__init__.py

@@ -2,11 +2,13 @@
 from AWSLibrary.keywords.s3 import S3Keywords
 from AWSLibrary.keywords.resource import ResourceKeywords
 from AWSLibrary.keywords.dynamo import DynamoKeywords
+from AWSLibrary.keywords.cloudWatch import CloudWatchKeywords
 
 
 __all__ = [
     SessionKeywords,
     S3Keywords,
     ResourceKeywords,
-    DynamoKeywords
+    DynamoKeywords,
+    CloudWatchKeywords
 ]

src/AWSLibrary/keywords/cloudWatch.py

@@ -0,0 +1,146 @@
+from AWSLibrary.librarycomponent import LibraryComponent
+from robot.api.deco import keyword
+from robot.api import logger
+from datetime import datetime, timedelta
+import time
+import re
+
+
+class CloudWatchKeywords(LibraryComponent):
+
+    def __init__(self, library):
+        LibraryComponent.__init__(self, library)
+        self.endpoint_url = None
+
+    @keyword('CloudWatch Set Endpoint Url')
+    def cloudwatch_set_endpoint(self, url):
+        """ The complete URL to use for the constructed CloudWatch client. Normally, botocore will automatically construct the
+        appropriate URL to use when communicating with a service. You can specify a complete URL
+        (including the “http/https” scheme) to override this behavior.
+
+        | =Arguments= | =Description= |
+        | ``url`` | <str> The complete endpoint URL. |
+
+        *Examples:*
+        | CloudWatch Set Endpoint Url | http://localhost:4566/ |
+        """
+        self.endpoint_url = url
+
+    @keyword('CloudWatch Logs Insights')
+    def insights_query(self, log_group, query, start_time=60):
+        """Executes a query on CloudWatch Insights and return the found results in a list.
+
+        | =Arguments= | =Description= |
+        | ``log_group`` | <str> Log group name. |
+        | ``query`` | <str> Aws query log format. |
+        | ``start_time`` | <str> The beginning of the time range to query from now to ago in minutes. |
+
+        ---
+        Use the same aws console ``query`` format in the argument, like this examples:
+
+        - Filter only by a part of the message, return the timestamp and the message:
+        | ``fields @timestamp, @message | filter @message like 'some string inside message to search' | sort @timestamp desc | limit 5``
+        - Filter by json path and part of the message, return only the message:
+        | ``fields @message | filter API.httpMethod = 'GET' and @message like 'Zp8beEeByQ0EDvg' | sort @timestamp desc | limit 20``
+        - Find the 10 most expensive requests:
+        | ``filter @type = "REPORT" | fields @requestId, @billedDuration | sort by @billedDuration desc | limit 10``
+
+        For more information, see CloudWatch Logs Insights Query Syntax.
+        https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CWL_QuerySyntax.html
+        ---
+
+        *Examples:*
+        | ${logs} | CloudWatch Logs Insights | /aws/group-name | query |
+        | ${logs} | CloudWatch Logs Insights | /aws/group-name | query | start_time=120 |
+        """
+        client = self.library.session.client('logs', endpoint_url=self.endpoint_url)
+        time_behind = (datetime.now() - timedelta(minutes=start_time)).timestamp()
+        query = client.start_query(logGroupName=log_group,
+                                   startTime=int(time_behind),
+                                   endTime=int(datetime.now().timestamp()),
+                                   queryString=query)
+        query_id = query['queryId']
+        response = client.get_query_results(queryId=query_id)
+        while response['status'] == 'Running':
+            logger.debug("waiting for Logs Insights")
+            time.sleep(0.5)
+            response = client.get_query_results(queryId=query_id)
+        return response['results']
+
+    @keyword('CloudWatch Wait For Logs')
+    def wait_for_logs(self, log_group, filter_pattern, regex_pattern, seconds_behind=60, timeout=30,
+                      not_found_fail=False):
+        """Wait until find the wanted log in cloudwatch.
+
+        This keyword is used to wait in real time if the desired log appears inside the informed log group.
+        It works in a similar way to the existing CloudWatch filter in "Live Tail".
+
+        Return all the logs that match the informed regex in a list.
+
+        | =Arguments= | =Description= |
+        | ``log_group`` | <str> Log group name. |
+        | ``filter_pattern`` | <str> Filter for CloudWatch. |
+        | ``regex_pattern`` | <str> Regex pattern to search in filter results. |
+        | ``seconds_behind`` | <str> How many seconds from now to ago, used to searching the logs. |
+        | ``timeout`` | <str> Timeout in seconds to end the search. |
+        | ``not_found_fail`` | <bool> If set as True, the keyword will fail if not find any log |
+
+        ---
+        For ``filter_pattern`` use the same as aws console filter patterns in Live tail.
+        https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html
+
+        - Filter for json path in log:
+        | {$.foo.bar = some_string_value}
+        - Filter for json path with null value in log:
+        | {$.foo.bar IS NULL}
+        - Filter for INFO logs:
+        | INFO
+        - Filter for DEBUG logs:
+        | DEBUG
+        - Filter for anything in logs:
+        | " "
+
+        For ``regex_pattern`` use the same regular expressions that robot framework uses in BuildIn Library.
+        ---
+
+        Note: as boto3 takes some time to get the logs and apply the regex query to each one of them, depending on the
+        amount of log found, the keyword execution time may be slightly longer than the timeout.
+
+        *Examples:*
+        | ${logs} | CloudWatch Wait For Logs | /aws/group_name | {$.foo.bar = id_value} | 2024.*filename |
+        | ${logs} | CloudWatch Wait For Logs | /aws/group_name | INFO | \\\d+.*id_code | timeout=60 |
+        | ${logs} | CloudWatch Wait For Logs | /aws/group_name | " " | \\\w+.*some_code | not_found_fail=${True} |
+        """
+        client = self.library.session.client('logs', endpoint_url=self.endpoint_url)
+        stream_response = client.describe_log_streams(logGroupName=log_group,
+                                                      orderBy='LastEventTime',
+                                                      descending=True,
+                                                      limit=1)
+        latest_log_stream_name = stream_response["logStreams"][0]["logStreamName"]
+        logger.info("The latest stream is: %s" % latest_log_stream_name)
+        stream_response = client.describe_log_streams(logGroupName=log_group,
+                                                      logStreamNamePrefix=latest_log_stream_name)
+        logger.debug(stream_response)
+        last_event = stream_response['logStreams'][0]['lastIngestionTime']
+        logger.info("Last event: %s" % datetime.fromtimestamp(int(last_event) / 1000).strftime('%d-%m-%Y %H:%M:%S'))
+        last_event_delay = last_event - seconds_behind * 1000
+        logger.info("Starting the log search from: %s" % datetime.fromtimestamp(int(last_event_delay) / 1000)
+                    .strftime('%d-%m-%Y %H:%M:%S'))
+        events_match = []
+        for i in range(int(timeout)):
+            response = client.filter_log_events(logGroupName=log_group,
+                                                startTime=last_event_delay,
+                                                filterPattern=filter_pattern)
+            logger.info("%s Total records found" % len(response["events"]))
+            logger.debug(response["events"])
+            for event in response["events"]:
+                match_event = re.search(regex_pattern, event['message'])
+                if match_event:
+                    events_match.append(event['message'])
+            if len(events_match) > 0:
+                break
+            else:
+                time.sleep(1)
+        if not_found_fail and len(events_match) == 0:
+            raise Exception(f"Log not found in CloudWatch inside {log_group} for {filter_pattern} and {regex_pattern}")
+        return events_match

src/AWSLibrary/keywords/dynamo.py

@@ -8,9 +8,25 @@ class DynamoKeywords(LibraryComponent):
 
     def __init__(self, library):
         LibraryComponent.__init__(self, library)
+        self.endpoint_url = None
+
+    @keyword('Dynamo Set Endpoint Url')
+    def dynamo_set_endpoint(self, url):
+        """ The complete URL to use for the constructed Dynamo client. Normally, botocore will automatically construct
+        the appropriate URL to use when communicating with a service. You can specify a complete URL
+        (including the “http/https” scheme) to override this behavior.
+
+        | =Arguments= | =Description= |
+        | ``url`` | <str> The complete endpoint URL. |
+
+        *Examples:*
+        | Dynamo Set Endpoint Url | http://localhost:4566/ |
+        """
+        self.endpoint_url = url
 
     @keyword('Dynamo Query Table')
-    def dynamo_query_table(self, table_name, partition_key, partition_value, sort_key=None, sort_value=None, projection=None):
+    def dynamo_query_table(self, table_name, partition_key, partition_value, sort_key=None, sort_value=None,
+                           projection=None):
         """Queries a DynamoDB table based on the partition_key and his value. returns all the information found in a
         list of dictionaries.
 
@@ -30,7 +46,7 @@ def dynamo_query_table(self, table_name, partition_key, partition_value, sort_ke
         | Dynamo Query Table | library-books | book_id | 123 | sort_key=book_code | sort_value=abc001 |
         | Dynamo Query Table | library-books | book_id | 123 | projection=value |
         """
-        client = self.library.session.client('dynamodb')
+        client = self.library.session.client('dynamodb', endpoint_url=self.endpoint_url)
         if sort_key is None:
             expression = {':value': {'S': partition_value}}
             condition = f'{partition_key} = :value'
@@ -69,7 +85,7 @@ def dynamo_update_item(self, table_name, json_dict):
         *Examples:*
         | Update Item | library-books | {"key": "value"} |
         """
-        resource = self.library.session.resource('dynamodb')
+        resource = self.library.session.resource('dynamodb', endpoint_url=self.endpoint_url)
         response = resource.Table(table_name).put_item(Item=json_dict)
         logger.info(response)
 
@@ -88,27 +104,30 @@ def dynamo_delete_item(self, table_name, partition_key, partition_value, sort_ke
         | Dynamo Delete Item | library-books | book_id | 123 |
         | Dynamo Delete Item | library-books | book_id | 123 | book_code | abc001 |
         """
-        resource = self.library.session.resource('dynamodb')
+        resource = self.library.session.resource('dynamodb', endpoint_url=self.endpoint_url)
         key = {partition_key: partition_value, sort_key: sort_value} if sort_key else {partition_key: partition_value}
         response = resource.Table(table_name).delete_item(Key=key)
         logger.info(response)
 
     @keyword('Dynamo Remove Key')
-    def dynamo_remove_key(self, table_name, partition_key, partition_value, attribute_name, sort_key=None, sort_value=None):
+    def dynamo_remove_key(self, table_name, partition_key, partition_value, attribute_name,
+                          sort_key=None, sort_value=None):
         """Removes a specific key in a DynamoDB item based on partition_key and sort key, if provided.
 
         | =Arguments= | =Description= |
         | ``table_name`` | <str> Name of the DynamoDB table. |
         | ``partition_key`` | <str> The key to search. |
         | ``partition_value`` | <str> Value of the partition key. |
         | ``attribute_name`` | <str> Key to remove, for nested keys use . to compose the path. |
+        | ``sort_key`` | <str> (optional) The sort key to search. |
+        | ``sort_value`` | <str> (optional) Value of the sort key. |
 
         *Examples:*
         | Dynamo Remove Key | library-books | book_id | 123 | quantity |
         | Dynamo Remove Key | library-books | book_id | 123 | book.value |
-        | Dynamo Remove Key | library-books | book_id | 123 | book | sort_key=book_code | sort_value=abc001 |
+        | Dynamo Remove Key | library-books | book_id | 123 | quantity | sort_key=book_code | sort_value=abc001 |
         """
-        resource = self.library.session.resource('dynamodb')
+        resource = self.library.session.resource('dynamodb', endpoint_url=self.endpoint_url)
         expression, names = self._compose_expression(attribute_name, remove=True)
         logger.debug(f"UpdateExpression: {expression}")
         logger.debug(f"ExpressionAttributeNames: {names}")
@@ -120,6 +139,47 @@ def dynamo_remove_key(self, table_name, partition_key, partition_value, attribut
         )
         logger.info(response)
 
+    @keyword('Dynamo Update Key')
+    def dynamo_update_key(self, table_name, partition_key, partition_value, attribute_name, attribute_value,
+                          sort_key=None, sort_value=None):
+        """Update a specific key in a DynamoDB item based on partition_key and sort key, if provided.
+
+        Arguments:
+        - ``table_name``: name of the DynamoDB table.
+        - ``partition_key``: the partition key to search.
+        - ``value``: the value of partition key.
+        - ``attribute_name``: the key to update. For nested keys, use . to compose the path
+        - ``new_value``: the new value of the attribute_name.
+        - ``sort_key``: the sort key to search. Default as None
+        - ``sort_value``: the value of sort key. Default as None
+
+        | =Arguments= | =Description= |
+        | ``table_name`` | <str> Name of the DynamoDB table. |
+        | ``partition_key`` | <str> The key to search. |
+        | ``partition_value`` | <str> Value of the partition key. |
+        | ``attribute_name`` | <str> Key to update. For nested keys, use . to compose the path. |
+        | ``attribute_value`` | <str> The new value of the attribute_name. |
+        | ``sort_key`` | <str> (optional) The sort key to search. |
+        | ``sort_value`` | <str> (optional) Value of the sort key. |
+
+        *Examples:*
+        | Dynamo Update Key | library-books | book_id | 123 | quantity | 100 |
+        | Dynamo Update Key | library-books | book_id | 123 | book.value | 15 |
+        | Dynamo Update Key | library-books | book_id | 123 | quantity | 100 | sort_key=book_code | sort_value=abc001 |
+        """
+        resource = self.library.session.resource('dynamodb', endpoint_url=self.endpoint_url)
+        expression, names = self._compose_expression(attribute_name)
+        logger.debug(f"UpdateExpression: {expression}")
+        logger.debug(f"ExpressionAttributeNames: {names}")
+        key = {partition_key: partition_value, sort_key: sort_value} if sort_key else {partition_key: partition_value}
+        result = resource.Table(table_name).update_item(
+            Key=key,
+            UpdateExpression=expression,
+            ExpressionAttributeNames=names,
+            ExpressionAttributeValues={':new_value': attribute_value}
+        )
+        return result
+
     @staticmethod
     def _compose_expression(attribute, remove=False):
         if "." not in attribute: