From e05901e21adb10ebd5a45feecc5d32e1dd619362 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 Oct 2021 13:02:32 +0200 Subject: [PATCH 1/3] refactor: expose jmespath powertools functions --- aws_lambda_powertools/logging/logger.py | 2 +- .../utilities/feature_flags/appconfig.py | 2 +- .../utilities/idempotency/persistence/base.py | 2 +- .../jmespath_utils/__init__.py} | 24 +++++++++++++++++-- .../utilities/jmespath_utils/envelopes.py | 8 +++++++ .../utilities/validation/validator.py | 3 ++- tests/functional/idempotency/conftest.py | 2 +- 7 files changed, 36 insertions(+), 7 deletions(-) rename aws_lambda_powertools/{shared/jmespath_utils.py => utilities/jmespath_utils/__init__.py} (66%) create mode 100644 aws_lambda_powertools/utilities/jmespath_utils/envelopes.py diff --git a/aws_lambda_powertools/logging/logger.py b/aws_lambda_powertools/logging/logger.py index e8b67a2ca7e..0b9b52f8824 100644 --- a/aws_lambda_powertools/logging/logger.py +++ b/aws_lambda_powertools/logging/logger.py @@ -446,7 +446,7 @@ def set_package_logger( ------- **Enables debug logging for AWS Lambda Powertools package** - >>> from aws_lambda_powertools.logging.logger import set_package_logger + >>> aws_lambda_powertools.logging.logger import set_package_logger >>> set_package_logger() Parameters diff --git a/aws_lambda_powertools/utilities/feature_flags/appconfig.py b/aws_lambda_powertools/utilities/feature_flags/appconfig.py index ff688dc6be5..dd581df9e22 100644 --- a/aws_lambda_powertools/utilities/feature_flags/appconfig.py +++ b/aws_lambda_powertools/utilities/feature_flags/appconfig.py @@ -4,10 +4,10 @@ from botocore.config import Config +from aws_lambda_powertools.utilities import jmespath_utils from aws_lambda_powertools.utilities.parameters import AppConfigProvider, GetParameterError, TransformParameterError from ... import Logger -from ...shared import jmespath_utils from .base import StoreProvider from .exceptions import ConfigurationStoreError, StoreClientError diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 4901e9f9f75..907af8edaa7 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -16,7 +16,6 @@ from aws_lambda_powertools.shared import constants from aws_lambda_powertools.shared.cache_dict import LRUDict -from aws_lambda_powertools.shared.jmespath_utils import PowertoolsFunctions from aws_lambda_powertools.shared.json_encoder import Encoder from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig from aws_lambda_powertools.utilities.idempotency.exceptions import ( @@ -25,6 +24,7 @@ IdempotencyKeyError, IdempotencyValidationError, ) +from aws_lambda_powertools.utilities.jmespath_utils import PowertoolsFunctions logger = logging.getLogger(__name__) diff --git a/aws_lambda_powertools/shared/jmespath_utils.py b/aws_lambda_powertools/utilities/jmespath_utils/__init__.py similarity index 66% rename from aws_lambda_powertools/shared/jmespath_utils.py rename to aws_lambda_powertools/utilities/jmespath_utils/__init__.py index bbb3b38fe04..a8d210bc1e0 100644 --- a/aws_lambda_powertools/shared/jmespath_utils.py +++ b/aws_lambda_powertools/utilities/jmespath_utils/__init__.py @@ -30,8 +30,27 @@ def _func_powertools_base64_gzip(self, value): return uncompressed.decode() -def extract_data_from_envelope(data: Union[Dict, str], envelope: str, jmespath_options: Optional[Dict]) -> Any: - """Searches data using JMESPath expression +def extract_data_from_envelope(data: Union[Dict, str], envelope: str, jmespath_options: Optional[Dict] = None) -> Any: + """Searches and extracts data using JMESPath + + Envelope being the JMESPath expression to extract the data you're after + + Built-in JMESPath functions include: powertools_json, powertools_base64, powertools_base64_gzip + + Examples + -------- + + **Deserialize JSON string and extracts data from body key** + + from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope + from aws_lambda_powertools.utilities.typing import LambdaContext + + + def handler(event: dict, context: LambdaContext): + # event = {"body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}"} # noqa: E800 + payload = extract_data_from_envelope(data=event, envelope="powertools_json(body)") + customer = payload.get("customerId") # now deserialized + ... Parameters ---------- @@ -42,6 +61,7 @@ def extract_data_from_envelope(data: Union[Dict, str], envelope: str, jmespath_o jmespath_options : Dict Alternative JMESPath options to be included when filtering expr + Returns ------- Any diff --git a/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py b/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py new file mode 100644 index 00000000000..df50e5f98d4 --- /dev/null +++ b/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py @@ -0,0 +1,8 @@ +API_GATEWAY_REST = "powertools_json(body)" +API_GATEWAY_HTTP = API_GATEWAY_REST +SQS = "Records[*].powertools_json(body)" +SNS = "Records[0].Sns.Message | powertools_json(@)" +EVENTBRIDGE = "detail" +CLOUDWATCH_EVENTS_SCHEDULED = EVENTBRIDGE +KINESIS_DATA_STREAM = "Records[*].kinesis.powertools_json(powertools_base64(data))" +CLOUDWATCH_LOGS = "awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]" diff --git a/aws_lambda_powertools/utilities/validation/validator.py b/aws_lambda_powertools/utilities/validation/validator.py index d9ce35fe41b..aab383eeb45 100644 --- a/aws_lambda_powertools/utilities/validation/validator.py +++ b/aws_lambda_powertools/utilities/validation/validator.py @@ -1,8 +1,9 @@ import logging from typing import Any, Callable, Dict, Optional, Union +from aws_lambda_powertools.utilities import jmespath_utils + from ...middleware_factory import lambda_handler_decorator -from ...shared import jmespath_utils from .base import validate_data_against_schema logger = logging.getLogger(__name__) diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index f563b4bbcda..71b5978497c 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -11,10 +11,10 @@ from botocore.config import Config from jmespath import functions -from aws_lambda_powertools.shared.jmespath_utils import extract_data_from_envelope from aws_lambda_powertools.shared.json_encoder import Encoder from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer from aws_lambda_powertools.utilities.idempotency.idempotency import IdempotencyConfig +from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope from aws_lambda_powertools.utilities.validation import envelopes from tests.functional.utils import load_event From 0d46d397c7ceffb6caa57d4e588cd4e910d72624 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 Oct 2021 13:03:18 +0200 Subject: [PATCH 2/3] docs(jmespath): expose jmespath powertools functions & extract fn --- docs/utilities/idempotency.md | 2 +- docs/utilities/jmespath_functions.md | 131 +++++++++++++++++++++++---- 2 files changed, 116 insertions(+), 17 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index bf06e3292b7..43eb1ac3a0b 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -209,7 +209,7 @@ Imagine the function executes successfully, but the client never receives the re !!! warning "Idempotency for JSON payloads" The payload extracted by the `event_key_jmespath` is treated as a string by default, so will be sensitive to differences in whitespace even when the JSON payload itself is identical. - To alter this behaviour, we can use the [JMESPath built-in function](/utilities/jmespath_functions) *powertools_json()* to treat the payload as a JSON object rather than a string. + To alter this behaviour, we can use the [JMESPath built-in function](jmespath_functions.md#powertools_json-function) `powertools_json()` to treat the payload as a JSON object rather than a string. === "payment.py" diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 7ef6b2b32b2..e11452cd5cc 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -3,9 +3,106 @@ title: JMESPath Functions description: Utility --- -You might have events or responses that contain non-encoded JSON, where you need to decode so that you can access portions of the object or ensure the Powertools utility receives a JSON object. This is a common use case when using the [validation](/utilities/validation) or [idempotency](/utilities/idempotency) utilities. +!!! tip "JMESPath is a query language for JSON used by AWS CLI, AWS Python SDK, and AWS Lambda Powertools for Python." -## Built-in JMESPath functions +Built-in [JMESPath](https://jmespath.org/){target="_blank"} Functions to easily deserialize common encoded JSON payloads in Lambda functions. + +## Key features + +* Deserialize JSON from JSON strings, base64, and compressed data +* Use JMESPath to extract and combine data recursively + +## Getting started + +You might have events that contains encoded JSON payloads as string, base64, or even in compressed format. It is a common use case to decode and extract them partially or fully as part of your Lambda function invocation. + +Lambda Powertools also have utilities like [validation](validation.md), [idempotency](idempotency.md), or [feature flags](feature_flags.md) where you might need to extract a portion of your data before using them. + +### Extracting data + +You can use the `extract_data_from_envelope` function along with any [JMESPath expression](https://jmespath.org/tutorial.html){target="_blank"}. + +=== "app.py" + + ```python hl_lines="1 7" + from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope + + from aws_lambda_powertools.utilities.typing import LambdaContext + + + def handler(event: dict, context: LambdaContext): + payload = extract_data_from_envelope(data=event, envelope="powertools_json(body)") + customer = payload.get("customerId") # now deserialized + ... + ``` + +=== "event.json" + + ```json + { + "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}" + } + ``` + +### Built-in envelopes + +We provide built-in envelopes for popular JMESPath expressions used when looking to decode/deserialize JSON objects within AWS Lambda Event Sources. + +=== "app.py" + + ```python hl_lines="1 7" + from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope, envelopes + + from aws_lambda_powertools.utilities.typing import LambdaContext + + + def handler(event: dict, context: LambdaContext): + payload = extract_data_from_envelope(data=event, envelope=envelopes.SNS) + customer = payload.get("customerId") # now deserialized + ... + ``` + +=== "event.json" + + ```json hl_lines="6" + { + "Records": [ + { + "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78", + "receiptHandle": "MessageReceiptHandle", + "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\",\"booking\":{\"id\":\"5b2c4803-330b-42b7-811a-c68689425de1\",\"reference\":\"ySz7oA\",\"outboundFlightId\":\"20c0d2f2-56a3-4068-bf20-ff7703db552d\"},\"payment\":{\"receipt\":\"https:\/\/pay.stripe.com\/receipts\/acct_1Dvn7pF4aIiftV70\/ch_3JTC14F4aIiftV700iFq2CHB\/rcpt_K7QsrFln9FgFnzUuBIiNdkkRYGxUL0X\",\"amount\":100}}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1523232000000", + "SenderId": "123456789012", + "ApproximateFirstReceiveTimestamp": "1523232000001" + }, + "messageAttributes": {}, + "md5OfBody": "7b270e59b47ff90a553787216d55d91d", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue", + "awsRegion": "us-east-1" + } + ] + } + ``` + +These are all built-in envelopes you can use along with their expression as a reference: + +Envelope | JMESPath expression +------------------------------------------------- | --------------------------------------------------------------------------------- +**`API_GATEWAY_REST`** | `powertools_json(body)` +**`API_GATEWAY_HTTP`** | `API_GATEWAY_REST` +**`SQS`** | `Records[*].powertools_json(body)` +**`SNS`** | `Records[0].Sns.Message | powertools_json(@)` +**`EVENTBRIDGE`** | `detail` +**`CLOUDWATCH_EVENTS_SCHEDULED`** | `EVENTBRIDGE` +**`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))` +**`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]` + +## Advanced + +### Built-in JMESPath functions You can use our built-in JMESPath functions within your expressions to do exactly that to decode JSON Strings, base64, and uncompress gzip data. !!! info @@ -134,33 +231,35 @@ This sample will decompress and decode base64 data, then use JMESPath pipeline e !!! warning This should only be used for advanced use cases where you have special formats not covered by the built-in functions. - This will **replace all provided built-in functions such as `powertools_json`, so you will no longer be able to use them**. - For special binary formats that you want to decode before applying JSON Schema validation, you can bring your own [JMESPath function](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} and any additional option via `jmespath_options` param. -=== "custom_jmespath_function.py" +In order to keep the built-in functions from Powertools, you can subclass from `PowertoolsFunctions`: - ```python hl_lines="2 6-10 14" - from aws_lambda_powertools.utilities.validation import validator - from jmespath import functions +=== "custom_jmespath_function.py" - import schemas + ```python hl_lines="2-3 6-9 11 17" + from aws_lambda_powertools.utilities.jmespath_utils import ( + PowertoolsFunctions, extract_data_from_envelope) + from jmespath.functions import signature - class CustomFunctions(functions.Functions): - @functions.signature({'types': ['string']}) + class CustomFunctions(PowertoolsFunctions): + @signature({'types': ['string']}) # Only decode if value is a string def _func_special_decoder(self, s): return my_custom_decoder_logic(s) custom_jmespath_options = {"custom_functions": CustomFunctions()} - @validator(schema=schemas.INPUT, jmespath_options=**custom_jmespath_options) def handler(event, context): - return event + # use the custom name after `_func_` + extract_data_from_envelope(data=event, + envelope="special_decoder(body)", + jmespath_options=**custom_jmespath_options) + ... ``` -=== "schemas.py" +=== "event.json" - ```python hl_lines="7 14 16 23 39 45 47 52" - --8<-- "docs/shared/validation_basic_jsonschema.py" + ```json + {"body": "custom_encoded_data"} ``` From 348aee25c5a021aa304bdbb9806f910d14b3f6ed Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 Oct 2021 13:03:36 +0200 Subject: [PATCH 3/3] chore: overdue linting Signed-off-by: heitorlessa --- .../feature_flags/test_feature_flags.py | 94 +++++++++++++++---- 1 file changed, 77 insertions(+), 17 deletions(-) diff --git a/tests/functional/feature_flags/test_feature_flags.py b/tests/functional/feature_flags/test_feature_flags.py index c421cc85423..8381dc6bf1d 100644 --- a/tests/functional/feature_flags/test_feature_flags.py +++ b/tests/functional/feature_flags/test_feature_flags.py @@ -802,7 +802,8 @@ def test_get_configuration_with_envelope_and_raw(mocker, config): assert "log_level" in config assert "log_level" not in features_config - + + ## ## Inequality test cases ## @@ -828,9 +829,12 @@ def test_flags_not_equal_no_match(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", context={"tenant_id": "345345435", "username": "a"}, default=False + ) assert toggle == expected_value + def test_flags_not_equal_match(mocker, config): expected_value = True mocked_app_config_schema = { @@ -876,9 +880,14 @@ def test_flags_less_than_no_match_1(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.12.25"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.12.25"}, + default=False, + ) assert toggle == expected_value + def test_flags_less_than_no_match_2(mocker, config): expected_value = False mocked_app_config_schema = { @@ -899,9 +908,14 @@ def test_flags_less_than_no_match_2(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.10.31"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.10.31"}, + default=False, + ) assert toggle == expected_value + def test_flags_less_than_match(mocker, config): expected_value = True mocked_app_config_schema = { @@ -922,10 +936,15 @@ def test_flags_less_than_match(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.04.01"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.04.01"}, + default=False, + ) assert toggle == expected_value -# Test less than or equal to + +# Test less than or equal to def test_flags_less_than_or_equal_no_match(mocker, config): expected_value = False mocked_app_config_schema = { @@ -946,9 +965,14 @@ def test_flags_less_than_or_equal_no_match(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.12.25"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.12.25"}, + default=False, + ) assert toggle == expected_value + def test_flags_less_than_or_equal_match_1(mocker, config): expected_value = True mocked_app_config_schema = { @@ -969,7 +993,11 @@ def test_flags_less_than_or_equal_match_1(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.04.01"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.04.01"}, + default=False, + ) assert toggle == expected_value @@ -993,9 +1021,14 @@ def test_flags_less_than_or_equal_match_2(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.10.31"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.10.31"}, + default=False, + ) assert toggle == expected_value + # Test greater than def test_flags_greater_than_no_match_1(mocker, config): expected_value = False @@ -1017,9 +1050,14 @@ def test_flags_greater_than_no_match_1(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.04.01"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.04.01"}, + default=False, + ) assert toggle == expected_value + def test_flags_greater_than_no_match_2(mocker, config): expected_value = False mocked_app_config_schema = { @@ -1040,9 +1078,14 @@ def test_flags_greater_than_no_match_2(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.10.31"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.10.31"}, + default=False, + ) assert toggle == expected_value + def test_flags_greater_than_match(mocker, config): expected_value = True mocked_app_config_schema = { @@ -1063,10 +1106,15 @@ def test_flags_greater_than_match(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.12.25"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.12.25"}, + default=False, + ) assert toggle == expected_value -# Test greater than or equal to + +# Test greater than or equal to def test_flags_greater_than_or_equal_no_match(mocker, config): expected_value = False mocked_app_config_schema = { @@ -1087,9 +1135,14 @@ def test_flags_greater_than_or_equal_no_match(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.04.01"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.04.01"}, + default=False, + ) assert toggle == expected_value + def test_flags_greater_than_or_equal_match_1(mocker, config): expected_value = True mocked_app_config_schema = { @@ -1110,7 +1163,11 @@ def test_flags_greater_than_or_equal_match_1(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.12.25"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.12.25"}, + default=False, + ) assert toggle == expected_value @@ -1134,6 +1191,9 @@ def test_flags_greater_than_or_equal_match_2(mocker, config): } } feature_flags = init_feature_flags(mocker, mocked_app_config_schema, config) - toggle = feature_flags.evaluate(name="my_feature", context={"tenant_id": "345345435", "username": "a", "current_date": "2021.10.31"}, default=False) + toggle = feature_flags.evaluate( + name="my_feature", + context={"tenant_id": "345345435", "username": "a", "current_date": "2021.10.31"}, + default=False, + ) assert toggle == expected_value -