diff --git a/aws_lambda_powertools/shared/constants.py b/aws_lambda_powertools/shared/constants.py index 622ffbce47b..45b46d236f9 100644 --- a/aws_lambda_powertools/shared/constants.py +++ b/aws_lambda_powertools/shared/constants.py @@ -21,3 +21,5 @@ XRAY_SDK_MODULE: str = "aws_xray_sdk" XRAY_SDK_CORE_MODULE: str = "aws_xray_sdk.core" + +IDEMPOTENCY_DISABLED_ENV: str = "POWERTOOLS_IDEMPOTENCY_DISABLED" diff --git a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py index d26144a262a..a4622fa9272 100644 --- a/aws_lambda_powertools/utilities/feature_flags/feature_flags.py +++ b/aws_lambda_powertools/utilities/feature_flags/feature_flags.py @@ -48,6 +48,10 @@ def _match_by_action(action: str, condition_value: Any, context_value: Any) -> b schema.RuleAction.ENDSWITH.value: lambda a, b: a.endswith(b), schema.RuleAction.IN.value: lambda a, b: a in b, schema.RuleAction.NOT_IN.value: lambda a, b: a not in b, + schema.RuleAction.KEY_IN_VALUE.value: lambda a, b: a in b, + schema.RuleAction.KEY_NOT_IN_VALUE.value: lambda a, b: a not in b, + schema.RuleAction.VALUE_IN_KEY.value: lambda a, b: b in a, + schema.RuleAction.VALUE_NOT_IN_KEY.value: lambda a, b: b not in a, } try: diff --git a/aws_lambda_powertools/utilities/feature_flags/schema.py b/aws_lambda_powertools/utilities/feature_flags/schema.py index efce82018db..68f8ebd9bca 100644 --- a/aws_lambda_powertools/utilities/feature_flags/schema.py +++ b/aws_lambda_powertools/utilities/feature_flags/schema.py @@ -22,6 +22,10 @@ class RuleAction(str, Enum): ENDSWITH = "ENDSWITH" IN = "IN" NOT_IN = "NOT_IN" + KEY_IN_VALUE = "KEY_IN_VALUE" + KEY_NOT_IN_VALUE = "KEY_NOT_IN_VALUE" + VALUE_IN_KEY = "VALUE_IN_KEY" + VALUE_NOT_IN_KEY = "VALUE_NOT_IN_KEY" class SchemaValidator(BaseValidator): @@ -80,7 +84,9 @@ class SchemaValidator(BaseValidator): The value MUST contain the following members: * **action**: `str`. Operation to perform to match a key and value. - The value MUST be either EQUALS, STARTSWITH, ENDSWITH, IN, NOT_IN + The value MUST be either EQUALS, STARTSWITH, ENDSWITH, + KEY_IN_VALUE KEY_NOT_IN_VALUE VALUE_IN_KEY VALUE_NOT_IN_KEY + * **key**: `str`. Key in given context to perform operation * **value**: `Any`. Value in given context that should match action operation. diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 06c9a578aa2..6984cfbbd8e 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -3,9 +3,11 @@ """ import functools import logging +import os from typing import Any, Callable, Dict, Optional, cast from aws_lambda_powertools.middleware_factory import lambda_handler_decorator +from aws_lambda_powertools.shared.constants import IDEMPOTENCY_DISABLED_ENV from aws_lambda_powertools.shared.types import AnyCallableT from aws_lambda_powertools.utilities.idempotency.base import IdempotencyHandler from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig @@ -56,6 +58,9 @@ def idempotent( >>> return {"StatusCode": 200} """ + if os.getenv(IDEMPOTENCY_DISABLED_ENV): + return handler(event, context) + config = config or IdempotencyConfig() args = event, context idempotency_handler = IdempotencyHandler( @@ -122,6 +127,9 @@ def process_order(customer_id: str, order: dict, **kwargs): @functools.wraps(function) def decorate(*args, **kwargs): + if os.getenv(IDEMPOTENCY_DISABLED_ENV): + return function(*args, **kwargs) + payload = kwargs.get(data_keyword_argument) if payload is None: diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index c1ed29c6fd3..0ce307ab503 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -62,11 +62,11 @@ def __init__( >>> return {"StatusCode": 200} """ - boto_config = boto_config or Config() - session = boto3_session or boto3.session.Session() - self._ddb_resource = session.resource("dynamodb", config=boto_config) + self._boto_config = boto_config or Config() + self._boto3_session = boto3_session or boto3.session.Session() + + self._table = None self.table_name = table_name - self.table = self._ddb_resource.Table(self.table_name) self.key_attr = key_attr self.expiry_attr = expiry_attr self.status_attr = status_attr @@ -74,6 +74,25 @@ def __init__( self.validation_key_attr = validation_key_attr super(DynamoDBPersistenceLayer, self).__init__() + @property + def table(self): + """ + Caching property to store boto3 dynamodb Table resource + + """ + if self._table: + return self._table + ddb_resource = self._boto3_session.resource("dynamodb", config=self._boto_config) + self._table = ddb_resource.Table(self.table_name) + return self._table + + @table.setter + def table(self, table): + """ + Allow table instance variable to be set directly, primarily for use in tests + """ + self._table = table + def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: """ Translate raw item records from DynamoDB to DataRecord @@ -125,7 +144,7 @@ def _put_record(self, data_record: DataRecord) -> None: ExpressionAttributeNames={"#id": self.key_attr, "#now": self.expiry_attr}, ExpressionAttributeValues={":now": int(now.timestamp())}, ) - except self._ddb_resource.meta.client.exceptions.ConditionalCheckFailedException: + except self.table.meta.client.exceptions.ConditionalCheckFailedException: logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") raise IdempotencyItemAlreadyExistsError diff --git a/docs/utilities/feature_flags.md b/docs/utilities/feature_flags.md index d22f9c03296..2836fb1759f 100644 --- a/docs/utilities/feature_flags.md +++ b/docs/utilities/feature_flags.md @@ -366,7 +366,7 @@ You can use `get_enabled_features` method for scenarios where you need a list of "when_match": true, "conditions": [ { - "action": "IN", + "action": "KEY_IN_VALUE", "key": "CloudFront-Viewer-Country", "value": ["NL", "IE", "UK", "PL", "PT"] } @@ -450,9 +450,20 @@ The `conditions` block is a list of conditions that contain `action`, `key`, and } ``` -The `action` configuration can have 5 different values: `EQUALS`, `STARTSWITH`, `ENDSWITH`, `IN`, `NOT_IN`. +The `action` configuration can have the following values, where the expressions **`a`** is the `key` and **`b`** is the `value` above: -The `key` and `value` will be compared to the input from the context parameter. +Action | Equivalent expression +------------------------------------------------- | --------------------------------------------------------------------------------- +**EQUALS** | `lambda a, b: a == b` +**STARTSWITH** | `lambda a, b: a.startswith(b)` +**ENDSWITH** | `lambda a, b: a.endswith(b)` +**KEY_IN_VALUE** | `lambda a, b: a in b` +**KEY_NOT_IN_VALUE** | `lambda a, b: a not in b` +**VALUE_IN_KEY** | `lambda a, b: b in a` +**VALUE_NOT_IN_KEY** | `lambda a, b: b not in a` + + +!!! info "The `**key**` and `**value**` will be compared to the input from the `**context**` parameter." **For multiple conditions**, we will evaluate the list of conditions as a logical `AND`, so all conditions needs to match to return `when_match` value. @@ -529,6 +540,27 @@ For this to work, you need to use a JMESPath expression via the `envelope` param } ``` +### Getting fetched configuration + +You can access the configuration fetched from the store via `get_raw_configuration` property within the store instance. + +=== "app.py" + + ```python hl_lines="12" + from aws_lambda_powertools.utilities.feature_flags import FeatureFlags, AppConfigStore + + app_config = AppConfigStore( + environment="dev", + application="product-catalogue", + name="configuration", + envelope = "feature_flags" + ) + + feature_flags = FeatureFlags(store=app_config) + + config = app_config.get_raw_configuration + ``` + ### Built-in store provider !!! info "For GA, you'll be able to bring your own store." @@ -650,3 +682,11 @@ Method | When to use | Requires new deployment on changes | Supported services **[Environment variables](https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html){target="_blank"}** | Simple configuration that will rarely if ever change, because changing it requires a Lambda function deployment. | Yes | Lambda **[Parameters utility](parameters.md)** | Access to secrets, or fetch parameters in different formats from AWS System Manager Parameter Store or Amazon DynamoDB. | No | Parameter Store, DynamoDB, Secrets Manager, AppConfig **Feature flags utility** | Rule engine to define when one or multiple features should be enabled depending on the input. | No | AppConfig + + +## Deprecation list when GA + +Breaking change | Recommendation +------------------------------------------------- | --------------------------------------------------------------------------------- +`IN` RuleAction | Use `KEY_IN_VALUE` instead +`NOT_IN` RuleAction | Use `KEY_NOT_IN_VALUE` instead diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 3508e2190e3..bf06e3292b7 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -765,6 +765,123 @@ The idempotency utility can be used with the `validator` decorator. Ensure that !!! tip "JMESPath Powertools functions are also available" Built-in functions known in the validation utility like `powertools_json`, `powertools_base64`, `powertools_base64_gzip` are also available to use in this utility. + +## Testing your code + +The idempotency utility provides several routes to test your code. + +### Disabling the idempotency utility +When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable `POWERTOOLS_IDEMPOTENCY_DISABLED` +with a truthy value. If you prefer setting this for specific tests, and are using Pytest, you can use [monkeypatch](https://docs.pytest.org/en/latest/monkeypatch.html) fixture: + +=== "tests.py" + + ```python hl_lines="2 3" + def test_idempotent_lambda_handler(monkeypatch): + # Set POWERTOOLS_IDEMPOTENCY_DISABLED before calling decorated functions + monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", 1) + + result = handler() + ... + ``` +=== "app.py" + + ```python + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, idempotent + ) + + persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency") + + @idempotent(persistence_store=persistence_layer) + def handler(event, context): + print('expensive operation') + return { + "payment_id": 12345, + "message": "success", + "statusCode": 200, + } + ``` + +### Testing with DynamoDB Local + +To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html), you can replace the `Table` resource used by the persistence layer with one you create inside your tests. This allows you to set the endpoint_url. + +=== "tests.py" + + ```python hl_lines="6 7 8" + import boto3 + + import app + + def test_idempotent_lambda(): + # Create our own Table resource using the endpoint for our DynamoDB Local instance + resource = boto3.resource("dynamodb", endpoint_url='http://localhost:8000') + table = resource.Table(app.persistence_layer.table_name) + app.persistence_layer.table = table + + result = app.handler({'testkey': 'testvalue'}, {}) + assert result['payment_id'] == 12345 + ``` + +=== "app.py" + + ```python + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, idempotent + ) + + persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency") + + @idempotent(persistence_store=persistence_layer) + def handler(event, context): + print('expensive operation') + return { + "payment_id": 12345, + "message": "success", + "statusCode": 200, + } + ``` + +### How do I mock all DynamoDB I/O operations + +The idempotency utility lazily creates the dynamodb [Table](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table) which it uses to access DynamoDB. +This means it is possible to pass a mocked Table resource, or stub various methods. + +=== "tests.py" + + ```python hl_lines="6 7 8 9" + from unittest.mock import MagicMock + + import app + + def test_idempotent_lambda(): + table = MagicMock() + app.persistence_layer.table = table + result = app.handler({'testkey': 'testvalue'}, {}) + table.put_item.assert_called() + ... + ``` + +=== "app.py" + + ```python + from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, idempotent + ) + + persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency") + + @idempotent(persistence_store=persistence_layer) + def handler(event, context): + print('expensive operation') + return { + "payment_id": 12345, + "message": "success", + "statusCode": 200, + } + ``` + ## Extra resources If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out diff --git a/tests/functional/feature_flags/test_feature_flags.py b/tests/functional/feature_flags/test_feature_flags.py index 8b6698a8179..f6ce93abaa9 100644 --- a/tests/functional/feature_flags/test_feature_flags.py +++ b/tests/functional/feature_flags/test_feature_flags.py @@ -301,6 +301,8 @@ def test_flags_conditions_rule_match_multiple_actions_multiple_rules_multiple_co # check a case where the feature exists but the rule doesn't match so we revert to the default value of the feature + +# Check IN/NOT_IN/KEY_IN_VALUE/KEY_NOT_IN_VALUE/VALUE_IN_KEY/VALUE_NOT_IN_KEY conditions def test_flags_match_rule_with_in_action(mocker, config): expected_value = True mocked_app_config_schema = { @@ -397,6 +399,207 @@ def test_flags_no_match_rule_with_not_in_action(mocker, config): assert toggle == expected_value +def test_flags_match_rule_with_key_in_value_action(mocker, config): + expected_value = True + mocked_app_config_schema = { + "my_feature": { + "default": False, + "rules": { + "tenant id is contained in [6, 2]": { + "when_match": expected_value, + "conditions": [ + { + "action": RuleAction.KEY_IN_VALUE.value, + "key": "tenant_id", + "value": ["6", "2"], + } + ], + } + }, + } + } + feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) + toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "6", "username": "a"}, default=False) + assert toggle == expected_value + + +def test_flags_no_match_rule_with_key_in_value_action(mocker, config): + expected_value = False + mocked_app_config_schema = { + "my_feature": { + "default": expected_value, + "rules": { + "tenant id is contained in [8, 2]": { + "when_match": True, + "conditions": [ + { + "action": RuleAction.KEY_IN_VALUE.value, + "key": "tenant_id", + "value": ["8", "2"], + } + ], + } + }, + } + } + feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) + toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "6", "username": "a"}, default=False) + assert toggle == expected_value + + +def test_flags_match_rule_with_key_not_in_value_action(mocker, config): + expected_value = True + mocked_app_config_schema = { + "my_feature": { + "default": False, + "rules": { + "tenant id is contained in [8, 2]": { + "when_match": expected_value, + "conditions": [ + { + "action": RuleAction.KEY_NOT_IN_VALUE.value, + "key": "tenant_id", + "value": ["10", "4"], + } + ], + } + }, + } + } + feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) + toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "6", "username": "a"}, default=False) + assert toggle == expected_value + + +def test_flags_no_match_rule_with_key_not_in_value_action(mocker, config): + expected_value = False + mocked_app_config_schema = { + "my_feature": { + "default": expected_value, + "rules": { + "tenant id is contained in [8, 2]": { + "when_match": True, + "conditions": [ + { + "action": RuleAction.KEY_NOT_IN_VALUE.value, + "key": "tenant_id", + "value": ["6", "4"], + } + ], + } + }, + } + } + feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) + toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "6", "username": "a"}, default=False) + assert toggle == expected_value + + +def test_flags_match_rule_with_value_in_key_action(mocker, config): + expected_value = True + mocked_app_config_schema = { + "my_feature": { + "default": False, + "rules": { + "user is in the SYSADMIN group": { + "when_match": expected_value, + "conditions": [ + { + "action": RuleAction.VALUE_IN_KEY.value, + "key": "groups", + "value": "SYSADMIN", + } + ], + } + }, + } + } + feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) + toggle = feature_flags.evaluate( + name="my_feature", context={"tenant_id": "6", "username": "a", "groups": ["SYSADMIN", "IT"]}, default=False + ) + assert toggle == expected_value + + +def test_flags_no_match_rule_with_value_in_key_action(mocker, config): + expected_value = False + mocked_app_config_schema = { + "my_feature": { + "default": expected_value, + "rules": { + "tenant id is contained in [8, 2]": { + "when_match": True, + "conditions": [ + { + "action": RuleAction.VALUE_IN_KEY.value, + "key": "groups", + "value": "GUEST", + } + ], + } + }, + } + } + feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) + toggle = feature_flags.evaluate( + name="my_feature", context={"tenant_id": "6", "username": "a", "groups": ["SYSADMIN", "IT"]}, default=False + ) + assert toggle == expected_value + + +def test_flags_match_rule_with_value_not_in_key_action(mocker, config): + expected_value = True + mocked_app_config_schema = { + "my_feature": { + "default": False, + "rules": { + "user is in the GUEST group": { + "when_match": expected_value, + "conditions": [ + { + "action": RuleAction.VALUE_NOT_IN_KEY.value, + "key": "groups", + "value": "GUEST", + } + ], + } + }, + } + } + feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) + toggle = feature_flags.evaluate( + name="my_feature", context={"tenant_id": "6", "username": "a", "groups": ["SYSADMIN", "IT"]}, default=False + ) + assert toggle == expected_value + + +def test_flags_no_match_rule_with_value_not_in_key_action(mocker, config): + expected_value = False + mocked_app_config_schema = { + "my_feature": { + "default": expected_value, + "rules": { + "user is in the SYSADMIN group": { + "when_match": True, + "conditions": [ + { + "action": RuleAction.VALUE_NOT_IN_KEY.value, + "key": "groups", + "value": "SYSADMIN", + } + ], + } + }, + } + } + feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) + toggle = feature_flags.evaluate( + name="my_feature", context={"tenant_id": "6", "username": "a", "groups": ["SYSADMIN", "IT"]}, default=False + ) + assert toggle == expected_value + + +# Check multiple features def test_multiple_features_enabled(mocker, config): expected_value = ["my_feature", "my_feature2"] mocked_app_config_schema = { diff --git a/tests/functional/feature_flags/test_schema_validation.py b/tests/functional/feature_flags/test_schema_validation.py index ce85494afce..1cd14aa4287 100644 --- a/tests/functional/feature_flags/test_schema_validation.py +++ b/tests/functional/feature_flags/test_schema_validation.py @@ -220,6 +220,26 @@ def test_valid_condition_all_actions(): CONDITION_KEY: "username", CONDITION_VALUE: ["c"], }, + { + CONDITION_ACTION: RuleAction.KEY_IN_VALUE.value, + CONDITION_KEY: "username", + CONDITION_VALUE: ["a", "b"], + }, + { + CONDITION_ACTION: RuleAction.KEY_NOT_IN_VALUE.value, + CONDITION_KEY: "username", + CONDITION_VALUE: ["c"], + }, + { + CONDITION_ACTION: RuleAction.VALUE_IN_KEY.value, + CONDITION_KEY: "groups", + CONDITION_VALUE: "SYSADMIN", + }, + { + CONDITION_ACTION: RuleAction.VALUE_NOT_IN_KEY.value, + CONDITION_KEY: "groups", + CONDITION_VALUE: "GUEST", + }, ], } }, diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index cb0d43ae6fa..b1d0914d181 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -3,6 +3,7 @@ import json import sys from hashlib import md5 +from unittest.mock import MagicMock import jmespath import pytest @@ -994,3 +995,25 @@ def dummy(payload): # WHEN dummy(payload=data_two) + + +def test_idempotency_disabled_envvar(monkeypatch, lambda_context, persistence_store: DynamoDBPersistenceLayer): + # Scenario to validate no requests sent to dynamodb table when 'POWERTOOLS_IDEMPOTENCY_DISABLED' is set + mock_event = {"data": "value"} + + persistence_store.table = MagicMock() + + monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", "1") + + @idempotent_function(data_keyword_argument="data", persistence_store=persistence_store) + def dummy(data): + return {"message": "hello"} + + @idempotent(persistence_store=persistence_store) + def dummy_handler(event, context): + return {"message": "hi"} + + dummy(data=mock_event) + dummy_handler(mock_event, lambda_context) + + assert len(persistence_store.table.method_calls) == 0