Merge branch 'develop' of https://github.com/awslabs/aws-lambda-power…

…tools-python into develop

* 'develop' of https://github.com/awslabs/aws-lambda-powertools-python:
  feat(feature-flags): improve "IN/NOT_IN"; new rule actions (aws-powertools#710)
  feat(idempotency): makes customers unit testing easier (aws-powertools#719)
  feat(feature-flags): get_raw_configuration property in Store (aws-powertools#720)

aws_lambda_powertools/shared/constants.py

@@ -21,3 +21,5 @@
 
 XRAY_SDK_MODULE: str = "aws_xray_sdk"
 XRAY_SDK_CORE_MODULE: str = "aws_xray_sdk.core"
+
+IDEMPOTENCY_DISABLED_ENV: str = "POWERTOOLS_IDEMPOTENCY_DISABLED"

aws_lambda_powertools/utilities/feature_flags/feature_flags.py

@@ -48,6 +48,10 @@ def _match_by_action(action: str, condition_value: Any, context_value: Any) -> b
             schema.RuleAction.ENDSWITH.value: lambda a, b: a.endswith(b),
             schema.RuleAction.IN.value: lambda a, b: a in b,
             schema.RuleAction.NOT_IN.value: lambda a, b: a not in b,
+            schema.RuleAction.KEY_IN_VALUE.value: lambda a, b: a in b,
+            schema.RuleAction.KEY_NOT_IN_VALUE.value: lambda a, b: a not in b,
+            schema.RuleAction.VALUE_IN_KEY.value: lambda a, b: b in a,
+            schema.RuleAction.VALUE_NOT_IN_KEY.value: lambda a, b: b not in a,
         }
 
         try:

aws_lambda_powertools/utilities/feature_flags/schema.py

@@ -22,6 +22,10 @@ class RuleAction(str, Enum):
     ENDSWITH = "ENDSWITH"
     IN = "IN"
     NOT_IN = "NOT_IN"
+    KEY_IN_VALUE = "KEY_IN_VALUE"
+    KEY_NOT_IN_VALUE = "KEY_NOT_IN_VALUE"
+    VALUE_IN_KEY = "VALUE_IN_KEY"
+    VALUE_NOT_IN_KEY = "VALUE_NOT_IN_KEY"
 
 
 class SchemaValidator(BaseValidator):
@@ -80,7 +84,9 @@ class SchemaValidator(BaseValidator):
     The value MUST contain the following members:
 
     * **action**: `str`. Operation to perform to match a key and value.
-    The value MUST be either EQUALS, STARTSWITH, ENDSWITH, IN, NOT_IN
+    The value MUST be either EQUALS, STARTSWITH, ENDSWITH,
+    KEY_IN_VALUE KEY_NOT_IN_VALUE VALUE_IN_KEY VALUE_NOT_IN_KEY
+
     * **key**: `str`. Key in given context to perform operation
     * **value**: `Any`. Value in given context that should match action operation.
 

aws_lambda_powertools/utilities/idempotency/idempotency.py

@@ -3,9 +3,11 @@
 """
 import functools
 import logging
+import os
 from typing import Any, Callable, Dict, Optional, cast
 
 from aws_lambda_powertools.middleware_factory import lambda_handler_decorator
+from aws_lambda_powertools.shared.constants import IDEMPOTENCY_DISABLED_ENV
 from aws_lambda_powertools.shared.types import AnyCallableT
 from aws_lambda_powertools.utilities.idempotency.base import IdempotencyHandler
 from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig
@@ -56,6 +58,9 @@ def idempotent(
         >>>     return {"StatusCode": 200}
     """
 
+    if os.getenv(IDEMPOTENCY_DISABLED_ENV):
+        return handler(event, context)
+
     config = config or IdempotencyConfig()
     args = event, context
     idempotency_handler = IdempotencyHandler(
@@ -122,6 +127,9 @@ def process_order(customer_id: str, order: dict, **kwargs):
 
     @functools.wraps(function)
     def decorate(*args, **kwargs):
+        if os.getenv(IDEMPOTENCY_DISABLED_ENV):
+            return function(*args, **kwargs)
+
         payload = kwargs.get(data_keyword_argument)
 
         if payload is None:

aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py

@@ -62,18 +62,37 @@ def __init__(
             >>>     return {"StatusCode": 200}
         """
 
-        boto_config = boto_config or Config()
-        session = boto3_session or boto3.session.Session()
-        self._ddb_resource = session.resource("dynamodb", config=boto_config)
+        self._boto_config = boto_config or Config()
+        self._boto3_session = boto3_session or boto3.session.Session()
+
+        self._table = None
         self.table_name = table_name
-        self.table = self._ddb_resource.Table(self.table_name)
         self.key_attr = key_attr
         self.expiry_attr = expiry_attr
         self.status_attr = status_attr
         self.data_attr = data_attr
         self.validation_key_attr = validation_key_attr
         super(DynamoDBPersistenceLayer, self).__init__()
 
+    @property
+    def table(self):
+        """
+        Caching property to store boto3 dynamodb Table resource
+
+        """
+        if self._table:
+            return self._table
+        ddb_resource = self._boto3_session.resource("dynamodb", config=self._boto_config)
+        self._table = ddb_resource.Table(self.table_name)
+        return self._table
+
+    @table.setter
+    def table(self, table):
+        """
+        Allow table instance variable to be set directly, primarily for use in tests
+        """
+        self._table = table
+
     def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord:
         """
         Translate raw item records from DynamoDB to DataRecord
@@ -125,7 +144,7 @@ def _put_record(self, data_record: DataRecord) -> None:
                 ExpressionAttributeNames={"#id": self.key_attr, "#now": self.expiry_attr},
                 ExpressionAttributeValues={":now": int(now.timestamp())},
             )
-        except self._ddb_resource.meta.client.exceptions.ConditionalCheckFailedException:
+        except self.table.meta.client.exceptions.ConditionalCheckFailedException:
             logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}")
             raise IdempotencyItemAlreadyExistsError
 

docs/utilities/feature_flags.md

@@ -366,7 +366,7 @@ You can use `get_enabled_features` method for scenarios where you need a list of
                     "when_match": true,
                     "conditions": [
                         {
-                            "action": "IN",
+                            "action": "KEY_IN_VALUE",
                             "key": "CloudFront-Viewer-Country",
                             "value": ["NL", "IE", "UK", "PL", "PT"]
                         }
@@ -450,9 +450,20 @@ The `conditions` block is a list of conditions that contain `action`, `key`, and
     }
     ```
 
-The `action` configuration can have 5 different values: `EQUALS`, `STARTSWITH`, `ENDSWITH`, `IN`, `NOT_IN`.
+The `action` configuration can have the following values, where the expressions **`a`** is the `key` and **`b`** is the `value` above:
 
-The `key` and `value` will be compared to the input from the context parameter.
+Action | Equivalent expression
+------------------------------------------------- | ---------------------------------------------------------------------------------
+**EQUALS** | `lambda a, b: a == b`
+**STARTSWITH** | `lambda a, b: a.startswith(b)`
+**ENDSWITH** | `lambda a, b: a.endswith(b)`
+**KEY_IN_VALUE** | `lambda a, b: a in b`
+**KEY_NOT_IN_VALUE** | `lambda a, b: a not in b`
+**VALUE_IN_KEY** | `lambda a, b: b in a`
+**VALUE_NOT_IN_KEY** | `lambda a, b: b not in a`
+
+
+!!! info "The `**key**` and `**value**` will be compared to the input from the `**context**` parameter."
 
 **For multiple conditions**, we will evaluate the list of conditions as a logical `AND`, so all conditions needs to match to return `when_match` value.
 
@@ -529,6 +540,27 @@ For this to work, you need to use a JMESPath expression via the `envelope` param
     }
     ```
 
+### Getting fetched configuration
+
+You can access the configuration fetched from the store via `get_raw_configuration` property within the store instance.
+
+=== "app.py"
+
+    ```python hl_lines="12"
+    from aws_lambda_powertools.utilities.feature_flags import FeatureFlags, AppConfigStore
+
+    app_config = AppConfigStore(
+        environment="dev",
+        application="product-catalogue",
+        name="configuration",
+        envelope = "feature_flags"
+    )
+
+	feature_flags = FeatureFlags(store=app_config)
+
+	config = app_config.get_raw_configuration
+    ```
+
 ### Built-in store provider
 
 !!! info "For GA, you'll be able to bring your own store."
@@ -650,3 +682,11 @@ Method | When to use | Requires new deployment on changes | Supported services
 **[Environment variables](https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html){target="_blank"}** | Simple configuration that will rarely if ever change, because changing it requires a Lambda function deployment. | Yes | Lambda
 **[Parameters utility](parameters.md)** | Access to secrets, or fetch parameters in different formats from AWS System Manager Parameter Store or Amazon DynamoDB. | No | Parameter Store, DynamoDB, Secrets Manager, AppConfig
 **Feature flags utility** | Rule engine to define when one or multiple features should be enabled depending on the input. | No | AppConfig
+
+
+## Deprecation list when GA
+
+Breaking change | Recommendation
+------------------------------------------------- | ---------------------------------------------------------------------------------
+`IN` RuleAction | Use `KEY_IN_VALUE` instead
+`NOT_IN` RuleAction | Use `KEY_NOT_IN_VALUE` instead

docs/utilities/idempotency.md

@@ -765,6 +765,123 @@ The idempotency utility can be used with the `validator` decorator. Ensure that
 !!! tip "JMESPath Powertools functions are also available"
     Built-in functions known in the validation utility like `powertools_json`, `powertools_base64`, `powertools_base64_gzip` are also available to use in this utility.
 
+
+## Testing your code
+
+The idempotency utility provides several routes to test your code.
+
+### Disabling the idempotency utility
+When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable `POWERTOOLS_IDEMPOTENCY_DISABLED`
+with a truthy value. If you prefer setting this for specific tests, and are using Pytest, you can use [monkeypatch](https://docs.pytest.org/en/latest/monkeypatch.html) fixture:
+
+=== "tests.py"
+
+    ```python hl_lines="2 3"
+    def test_idempotent_lambda_handler(monkeypatch):
+        # Set POWERTOOLS_IDEMPOTENCY_DISABLED before calling decorated functions
+        monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", 1)
+
+        result = handler()
+        ...
+    ```
+=== "app.py"
+
+    ```python
+    from aws_lambda_powertools.utilities.idempotency import (
+    DynamoDBPersistenceLayer, idempotent
+    )
+
+    persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency")
+
+    @idempotent(persistence_store=persistence_layer)
+    def handler(event, context):
+        print('expensive operation')
+        return {
+            "payment_id": 12345,
+            "message": "success",
+            "statusCode": 200,
+        }
+    ```
+
+### Testing with DynamoDB Local
+
+To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html), you can replace the `Table` resource used by the persistence layer with one you create inside your tests. This allows you to set the endpoint_url.
+
+=== "tests.py"
+
+    ```python hl_lines="6 7 8"
+    import boto3
+
+    import app
+
+    def test_idempotent_lambda():
+        # Create our own Table resource using the endpoint for our DynamoDB Local instance
+        resource = boto3.resource("dynamodb", endpoint_url='http://localhost:8000')
+        table = resource.Table(app.persistence_layer.table_name)
+        app.persistence_layer.table = table
+
+        result = app.handler({'testkey': 'testvalue'}, {})
+        assert result['payment_id'] == 12345
+    ```
+
+=== "app.py"
+
+    ```python
+    from aws_lambda_powertools.utilities.idempotency import (
+    DynamoDBPersistenceLayer, idempotent
+    )
+
+    persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency")
+
+    @idempotent(persistence_store=persistence_layer)
+    def handler(event, context):
+        print('expensive operation')
+        return {
+            "payment_id": 12345,
+            "message": "success",
+            "statusCode": 200,
+        }
+    ```
+
+### How do I mock all DynamoDB I/O operations
+
+The idempotency utility lazily creates the dynamodb [Table](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table) which it uses to access DynamoDB.
+This means it is possible to pass a mocked Table resource, or stub various methods.
+
+=== "tests.py"
+
+    ```python hl_lines="6 7 8 9"
+    from unittest.mock import MagicMock
+
+    import app
+
+    def test_idempotent_lambda():
+        table = MagicMock()
+        app.persistence_layer.table = table
+        result = app.handler({'testkey': 'testvalue'}, {})
+        table.put_item.assert_called()
+        ...
+    ```
+
+=== "app.py"
+
+    ```python
+    from aws_lambda_powertools.utilities.idempotency import (
+    DynamoDBPersistenceLayer, idempotent
+    )
+
+    persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency")
+
+    @idempotent(persistence_store=persistence_layer)
+    def handler(event, context):
+        print('expensive operation')
+        return {
+            "payment_id": 12345,
+            "message": "success",
+            "statusCode": 200,
+        }
+    ```
+
 ## Extra resources
 
 If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out