[Serverless] Allow some keys to be option in serverless integration t…

…ests. (#14827) * Ability to remove items from the json. * Remove items from snapshot.
DataDog · Dec 21, 2022 · fec7ca8 · fec7ca8
1 parent db79950
commit fec7ca8
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 37 deletions.
diff --git a/test/integration/serverless/log_normalize.py b/test/integration/serverless/log_normalize.py
@@ -42,13 +42,15 @@ def normalize_logs(stage):
         replace(stage, 'XXXXXX'),
         replace(r'(architecture:)(x86_64|arm64)', r'\1XXX'),
         sort_by(lambda log: log["message"]["message"]),
-        # ignore a Lambda error that occurs sporadically for log-csharp see here for more info:
-        # https://repost.aws/questions/QUq2OfIFUNTCyCKsChfJLr5w/lambda-function-working-locally-but-crashing-on-aws
-        # TODO
-        # perl -n -e "print unless /LAMBDA_RUNTIME Failed to get next invocation. No Response from endpoint/ or \
-        #  /An error occurred while attempting to execute your code.: LambdaException/ or \
-        #  /terminate called after throwing an instance of 'std::logic_error'/ or \
-        #  /basic_string::_M_construct null not valid/" |
+        # TODO: these messages may be an indication of a real problem and
+        # should be investigated
+        rm_item(
+            lambda log: log["message"]["message"]
+            in (
+                "TIMESTAMP UTC | DD_EXTENSION | ERROR | could not forward the request context canceled\n",
+                "TIMESTAMP http: proxy error: context canceled\n",
+            )
+        ),
     ]
 
 
@@ -79,6 +81,9 @@ def normalize_traces(stage):
 
 
 def replace(pattern, repl):
+    """
+    Replace all substrings matching regex pattern with given replacement string
+    """
     comp = re.compile(pattern, flags=re.DOTALL)
 
     def _replace(log):
@@ -88,10 +93,16 @@ def _replace(log):
 
 
 def exclude(pattern):
+    """
+    Remove all substrings matching regex pattern
+    """
     return replace(pattern, '')
 
 
 def require(pattern):
+    """
+    Remove all substrings that don't match the given regex pattern
+    """
     comp = re.compile(pattern, flags=re.DOTALL)
 
     def _require(log):
@@ -104,6 +115,11 @@ def _require(log):
 
 
 def sort_by(key):
+    """
+    Sort the json entries using the given key function, requires the log string
+    to be proper json and to be a list
+    """
+
     def _sort(log):
         log_json = json.loads(log, strict=False)
         log_sorted = sorted(log_json, key=key)
@@ -112,6 +128,21 @@ def _sort(log):
     return _sort
 
 
+def rm_item(key):
+    """
+    Delete json entries from the log string using the given key function, key
+    takes an item from the json list and must return boolean which is True when
+    the item is to be removed and False if it is to be kept
+    """
+
+    def _rm_item(log):
+        log_json = json.loads(log, strict=False)
+        new_logs = [i for i in log_json if not key(i)]
+        return json.dumps(new_logs)
+
+    return _rm_item
+
+
 ###################
 # END NORMALIZERS #
 ###################

diff --git a/test/integration/serverless/snapshots/log-proxy b/test/integration/serverless/snapshots/log-proxy
@@ -178,35 +178,5 @@
     "service": "integration-tests-service",
     "ddsource": "lambda",
     "ddtags": "account_id:425362996713,architecture:XXX,aws_account:425362996713,dd_extension_version:123,env:integration-tests-env,function_arn:arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy,functionname:integration-tests-extension-XXXXXX-log-proxy,memorysize:1024,region:eu-west-1,resource:integration-tests-extension-XXXXXX-log-proxy,runtime:nodejs14.x,service:integration-tests-service,taga:valuea,tagb:valueb,tagc:valuec,tagd:valued,version:integration-tests-version"
-  },
-  {
-    "message": {
-      "message": "TIMESTAMP UTC | DD_EXTENSION | ERROR | could not forward the request context canceled\n",
-      "lambda": {
-        "arn": "arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy",
-        "request_id": "XXX"
-      }
-    },
-    "status": "info",
-    "timestamp": "XXX",
-    "hostname": "arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy",
-    "service": "integration-tests-service",
-    "ddsource": "lambda",
-    "ddtags": "account_id:425362996713,architecture:XXX,aws_account:425362996713,dd_extension_version:123,env:integration-tests-env,function_arn:arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy,functionname:integration-tests-extension-XXXXXX-log-proxy,memorysize:1024,region:eu-west-1,resource:integration-tests-extension-XXXXXX-log-proxy,runtime:nodejs14.x,service:integration-tests-service,taga:valuea,tagb:valueb,tagc:valuec,tagd:valued,version:integration-tests-version"
-  },
-  {
-    "message": {
-      "message": "TIMESTAMP http: proxy error: context canceled\n",
-      "lambda": {
-        "arn": "arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy",
-        "request_id": "XXX"
-      }
-    },
-    "status": "info",
-    "timestamp": "XXX",
-    "hostname": "arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy",
-    "service": "integration-tests-service",
-    "ddsource": "lambda",
-    "ddtags": "account_id:425362996713,architecture:XXX,aws_account:425362996713,dd_extension_version:123,env:integration-tests-env,function_arn:arn:aws:lambda:eu-west-1:425362996713:function:integration-tests-extension-XXXXXX-log-proxy,functionname:integration-tests-extension-XXXXXX-log-proxy,memorysize:1024,region:eu-west-1,resource:integration-tests-extension-XXXXXX-log-proxy,runtime:nodejs14.x,service:integration-tests-service,taga:valuea,tagb:valueb,tagc:valuec,tagd:valued,version:integration-tests-version"
   }
 ]