From fec7ca837b0eb33c188b206f962d09adf0f516cf Mon Sep 17 00:00:00 2001 From: Rey Abolofia Date: Wed, 21 Dec 2022 15:15:49 -0700 Subject: [PATCH] [Serverless] Allow some keys to be option in serverless integration tests. (#14827) * Ability to remove items from the json. * Remove items from snapshot. --- test/integration/serverless/log_normalize.py | 45 ++++++++++++++++--- .../serverless/snapshots/log-proxy | 30 ------------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/test/integration/serverless/log_normalize.py b/test/integration/serverless/log_normalize.py index 5a687e18f9237..c018983ad9130 100644 --- a/test/integration/serverless/log_normalize.py +++ b/test/integration/serverless/log_normalize.py @@ -42,13 +42,15 @@ def normalize_logs(stage): replace(stage, 'XXXXXX'), replace(r'(architecture:)(x86_64|arm64)', r'\1XXX'), sort_by(lambda log: log["message"]["message"]), - # ignore a Lambda error that occurs sporadically for log-csharp see here for more info: - # https://repost.aws/questions/QUq2OfIFUNTCyCKsChfJLr5w/lambda-function-working-locally-but-crashing-on-aws - # TODO - # perl -n -e "print unless /LAMBDA_RUNTIME Failed to get next invocation. No Response from endpoint/ or \ - # /An error occurred while attempting to execute your code.: LambdaException/ or \ - # /terminate called after throwing an instance of 'std::logic_error'/ or \ - # /basic_string::_M_construct null not valid/" | + # TODO: these messages may be an indication of a real problem and + # should be investigated + rm_item( + lambda log: log["message"]["message"] + in ( + "TIMESTAMP UTC | DD_EXTENSION | ERROR | could not forward the request context canceled\n", + "TIMESTAMP http: proxy error: context canceled\n", + ) + ), ] @@ -79,6 +81,9 @@ def normalize_traces(stage): def replace(pattern, repl): + """ + Replace all substrings matching regex pattern with given replacement string + """ comp = re.compile(pattern, flags=re.DOTALL) def _replace(log): @@ -88,10 +93,16 @@ def _replace(log): def exclude(pattern): + """ + Remove all substrings matching regex pattern + """ return replace(pattern, '') def require(pattern): + """ + Remove all substrings that don't match the given regex pattern + """ comp = re.compile(pattern, flags=re.DOTALL) def _require(log): @@ -104,6 +115,11 @@ def _require(log): def sort_by(key): + """ + Sort the json entries using the given key function, requires the log string + to be proper json and to be a list + """ + def _sort(log): log_json = json.loads(log, strict=False) log_sorted = sorted(log_json, key=key) @@ -112,6 +128,21 @@ def _sort(log): return _sort +def rm_item(key): + """ + Delete json entries from the log string using the given key function, key + takes an item from the json list and must return boolean which is True when + the item is to be removed and False if it is to be kept + """ + + def _rm_item(log): + log_json = json.loads(log, strict=False) + new_logs = [i for i in log_json if not key(i)] + return json.dumps(new_logs) + + return _rm_item + + ################### # END NORMALIZERS # ################### diff --git a/test/integration/serverless/snapshots/log-proxy b/test/integration/serverless/snapshots/log-proxy index d6c146d4f4977..283aa1a2633c4 100644 --- a/test/integration/serverless/snapshots/log-proxy +++ b/test/integration/serverless/snapshots/log-proxy @@ -178,35 +178,5 @@ "service": "integration-tests-service", "ddsource": "lambda", "ddtags": "account_id:425362996713,architecture:XXX,aws_account:425362996713,dd_extension_version:123,env:integration-tests-env,function_arn:arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy,functionname:integration-tests-extension-XXXXXX-log-proxy,memorysize:1024,region:eu-west-1,resource:integration-tests-extension-XXXXXX-log-proxy,runtime:nodejs14.x,service:integration-tests-service,taga:valuea,tagb:valueb,tagc:valuec,tagd:valued,version:integration-tests-version" - }, - { - "message": { - "message": "TIMESTAMP UTC | DD_EXTENSION | ERROR | could not forward the request context canceled\n", - "lambda": { - "arn": "arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy", - "request_id": "XXX" - } - }, - "status": "info", - "timestamp": "XXX", - "hostname": "arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy", - "service": "integration-tests-service", - "ddsource": "lambda", - "ddtags": "account_id:425362996713,architecture:XXX,aws_account:425362996713,dd_extension_version:123,env:integration-tests-env,function_arn:arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy,functionname:integration-tests-extension-XXXXXX-log-proxy,memorysize:1024,region:eu-west-1,resource:integration-tests-extension-XXXXXX-log-proxy,runtime:nodejs14.x,service:integration-tests-service,taga:valuea,tagb:valueb,tagc:valuec,tagd:valued,version:integration-tests-version" - }, - { - "message": { - "message": "TIMESTAMP http: proxy error: context canceled\n", - "lambda": { - "arn": "arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy", - "request_id": "XXX" - } - }, - "status": "info", - "timestamp": "XXX", - "hostname": "arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy", - "service": "integration-tests-service", - "ddsource": "lambda", - "ddtags": "account_id:425362996713,architecture:XXX,aws_account:425362996713,dd_extension_version:123,env:integration-tests-env,function_arn:arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy,functionname:integration-tests-extension-XXXXXX-log-proxy,memorysize:1024,region:eu-west-1,resource:integration-tests-extension-XXXXXX-log-proxy,runtime:nodejs14.x,service:integration-tests-service,taga:valuea,tagb:valueb,tagc:valuec,tagd:valued,version:integration-tests-version" } ]