feat: adding _OP_FAILED_KEYS and FAILED_KEYS

CHANGELOG.md

@@ -10,6 +10,16 @@
 - `_OP_CLOUD_SYS_VENDOR` key for reporting sys vendor
   file content used to identity cloud provider.
   ([#418](https://github.com/crashappsec/chalk/pull/418))
+- `FAILED_KEYS` and `_OP_FAILED_KEYS` - metadata keys
+  which chalk could not collect metadata for.
+  Each key contains:
+
+  - `code` - short identifiable code of a known error
+  - `message` - exact encountered error/exception message
+  - `description` - human-readable description of the error
+    with additional context how to potentially resolve it
+
+  ([#422](https://github.com/crashappsec/chalk/pull/422))
 
 ## 0.4.12
 

src/chalk_common.nim

@@ -38,6 +38,7 @@ type
     opFailed*:      bool
     marked*:        bool
     embeds*:        seq[ChalkObj]
+    failedKeys*:    ChalkDict
     err*:           seq[string]      ## Runtime logs for chalking are filtered
                                      ## based on the "chalk log level". They
                                      ## end up here, until the end of chalking
@@ -464,6 +465,7 @@ let
 
 var
   hostInfo*               = ChalkDict()
+  failedKeys*             = ChalkDict()
   subscribedKeys*         = Table[string, bool]()
   systemErrors*           = seq[string](@[])
   selfChalk*              = ChalkObj(nil)

src/configs/base_keyspecs.c4m

@@ -1490,6 +1490,22 @@ variable that controls console logging output.
 """
 }
 
+keyspec FAILED_KEYS {
+  kind:     ChalkTimeArtifact
+  type:     dict[string, `x]
+  standard: true
+  since:    "0.4.13"
+  shortdoc: "Keys and errors why chalk could not collect them"
+  doc:      """
+In some cases there are known reasons why chalk cannot collect
+some metadata. For example chalk might not be able to collect
+AWS EC2 instance metadata if IMDSv2 is disabled for that instance.
+This key reports all known errors and reason why that key could
+not be collected so that further action can be taken to be able
+to collect that metadata in the future.
+"""
+}
+
 keyspec SIGNING {
     kind:             ChalkTimeArtifact
     never_early:      true
@@ -4503,6 +4519,22 @@ shares the same log-level configuration.
 """
 }
 
+keyspec _OP_FAILED_KEYS {
+  kind:     RunTimeHost
+  type:     dict[string, `x]
+  standard: true
+  since:    "0.4.13"
+  shortdoc: "Keys and errors why chalk could not collect them"
+  doc:      """
+In some cases there are known reasons why chalk cannot collect
+some metadata. For example chalk might not be able to collect
+AWS EC2 instance metadata if IMDSv2 is disabled for that instance.
+This key reports all known errors and reason why that key could
+not be collected so that further action can be taken to be able
+to collect that metadata in the future.
+"""
+}
+
 keyspec _OP_HOST_REPORT_KEYS {
     kind:             RunTimeHost
     type:             list[string]

src/configs/base_plugins.c4m

@@ -59,10 +59,10 @@ are not overridable via other plugins.
 
 plugin metsys {
     ~enabled: true
-    pre_chalk_keys:  ["METADATA_HASH", "ERR_INFO", "METADATA_ID",
+    pre_chalk_keys:  ["METADATA_HASH", "ERR_INFO", "FAILED_KEYS", "METADATA_ID",
                       "SIGNING", "SIGNATURE", "INJECTOR_PUBLIC_KEY"]
     post_chalk_keys: ["_SIGNATURES"]
-    post_run_keys:   ["_OP_ERRORS", "_CHALK_EXTERNAL_ACTION_AUDIT",
+    post_run_keys:   ["_OP_ERRORS", "_OP_FAILED_KEYS", "_CHALK_EXTERNAL_ACTION_AUDIT",
                       "_CHALK_RUN_TIME", "_OP_EXIT_CODE"]
     ~priority:       high()
     doc: """

src/configs/base_report_templates.c4m

@@ -362,6 +362,8 @@ report and subtract from it.
   key._OP_HOST_VERSION.use                    = true
   key._OP_HOST_MACHINE.use                    = true
   key._OP_CLOUD_METADATA.use                  = true
+  key._OP_FAILED_KEYS.use                     = true
+  key.FAILED_KEYS.use                         = true
   key._OP_ERRORS.use                          = true
   key._OP_HOST_REPORT_KEYS.use                = true
   key._OP_TCP_SOCKET_INFO.use                 = true
@@ -513,6 +515,8 @@ doc: """
   key._OP_HOST_VERSION.use                    = true
   key._OP_HOST_MACHINE.use                    = true
   key._OP_CLOUD_METADATA.use                  = true
+  key._OP_FAILED_KEYS.use                     = true
+  key.FAILED_KEYS.use                         = true
   key._OP_ERRORS.use                          = true
   key._OP_HOST_REPORT_KEYS.use                = true
   key._OP_TCP_SOCKET_INFO.use                 = true
@@ -998,6 +1002,8 @@ container.
   key._OP_HOST_VERSION.use                    = true
   key._OP_HOST_MACHINE.use                    = true
   key._OP_CLOUD_METADATA.use                  = true
+  key._OP_FAILED_KEYS.use                     = true
+  key.FAILED_KEYS.use                         = true
   key._CHALK_EXTERNAL_ACTION_AUDIT.use        = true
   key._OP_ERRORS.use                          = true
   key._OP_CLOUD_SYS_VENDOR.use                = true
@@ -1459,6 +1465,8 @@ and keep the run-time key.
   key._OP_HOST_VERSION.use                    = true
   key._OP_HOST_MACHINE.use                    = true
   key._OP_CLOUD_METADATA.use                  = true
+  key._OP_FAILED_KEYS.use                     = true
+  key.FAILED_KEYS.use                         = true
   key._CHALK_EXTERNAL_ACTION_AUDIT.use        = true
   key._OP_ERRORS.use                          = true
   key._OP_CLOUD_SYS_VENDOR.use                = true

src/configs/crashoverride.c4m

@@ -75,6 +75,8 @@ This is mostly a copy of insert template however all keys are immutable.
   ~key._OP_HOST_VERSION.use                           = true
   ~key._OP_HOST_MACHINE.use                           = true
   ~key._OP_CLOUD_METADATA.use                         = true
+  ~key._OP_FAILED_KEYS.use                            = true
+  ~key.FAILED_KEYS.use                                = true
   ~key._CHALK_EXTERNAL_ACTION_AUDIT.use               = true
   ~key._OP_ERRORS.use                                 = true
   ~key._OP_CLOUD_SYS_VENDOR.use                       = true

src/plugins/cloudMetadata.nim

@@ -200,14 +200,37 @@ proc getAwsToken(): Option[string] =
       body     = response.body().strip()
 
     if not response.code.is2xx():
+      if response.code == Http403:
+        addFailedKey(
+          "_OP_CLOUD_METADATA",
+          code = "IMDS_DISABLED",
+          error = response.status,
+          description = (
+            "IMDSv2 returned 403 Forbidden which implies IMDS metadata is disabled. " &
+            "Enable IMDSv2 for chalk to collect more information about EC2 instances. " &
+            "See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html#instance-metadata-returns."
+          ),
+        )
       trace("Could not retrieve IMDSv2 token from: " & url & " - " & response.status & ": " & body)
       return none(string)
 
     trace("Retrieved AWS metadata token")
     return some(body)
   except:
-    trace("Could not retrieve IMDSv2 token from: " & url & " due to: " & getCurrentExceptionMsg())
+    let msg = getCurrentExceptionMsg()
+    trace("Could not retrieve IMDSv2 token from: " & url & " due to: " & msg)
     dumpExOnDebug()
+    if "'readLine' timed out" in msg:
+      addFailedKey(
+        "_OP_CLOUD_METADATA",
+        code = "IMDS_HOP_LIMIT",
+        error = msg,
+        description = (
+          "Chalk timed out receiving response from IMDSv2 which implies chalk reached its hop limit. " &
+          "Hop limit needs to be at least 2 in order for a docker container to be able to receive response from IMDSv2. " &
+          "See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-options.html"
+        ),
+      )
     return none(string)
 
 proc oneItem(chalkDict: ChalkDict, token: string, keyname: string, url: string) =

src/plugins/system.nim

@@ -282,8 +282,8 @@ proc metsysGetChalkTimeArtifactInfo*(self: Plugin, obj: ChalkObj):
 
   # We add these directly into collectedData so that it can get
   # added to the MD hash when we call normalizeChalk()
-  if len(obj.err) != 0:
-    obj.collectedData["ERR_INFO"] = pack(obj.err)
+  obj.collectedData.setIfNeeded("ERR_INFO", obj.err)
+  obj.collectedData.setIfNeeded("FAILED_KEYS", obj.failedKeys)
 
   let
     toHash       = obj.getChalkMark().normalizeChalk()
@@ -314,12 +314,9 @@ proc metsysGetRunTimeHostInfo(self: Plugin, objs: seq[ChalkObj]):
   result = ChalkDict()
 
   result.setIfNeeded("_OP_EXIT_CODE", getExitCode())
-
-  if len(systemErrors)  != 0:
-    result.setIfNeeded("_OP_ERRORS", systemErrors)
-
-  if len(externalActions) > 0:
-    result.setIfNeeded("_CHALK_EXTERNAL_ACTION_AUDIT", externalActions)
+  result.setIfNeeded("_OP_ERRORS", systemErrors)
+  result.setIfNeeded("_OP_FAILED_KEYS", failedKeys)
+  result.setIfNeeded("_CHALK_EXTERNAL_ACTION_AUDIT", externalActions)
 
   if isSubscribedKey("_CHALK_RUN_TIME"):
     # startTime lives in runManagement.

src/run_management.nim

@@ -80,6 +80,7 @@ proc clearReportingState*() =
   hostInfo       = ChalkDict()
   subscribedKeys = Table[string, bool]()
   systemErrors   = @[]
+  failedKeys     = ChalkDict()
 
 proc pushCollectionCtx*(): CollectionCtx =
   result = CollectionCtx()
@@ -152,7 +153,9 @@ proc newChalk*(name:         string            = "",
                     resourceType:  resourceType,
                     extract:       extract,
                     cache:         cache,
-                    myCodec:       codec)
+                    myCodec:       codec,
+                    failedKeys:    ChalkDict(),
+                   )
 
   if chalkId != "":
     result.collectedData["CHALK_ID"] = pack(chalkId)
@@ -218,6 +221,17 @@ template trySetIfNeeded*(o: ChalkDict, k: string, code: untyped) =
 proc isChalkingOp*(): bool =
   return commandName in attrGet[seq[string]]("valid_chalk_command_names")
 
+proc addFailedKey*(key: string, code: string, error: string, description: string) =
+  let errObject = getErrorObject()
+  var failure   = ChalkDict()
+  failure["code"] = pack(code)
+  failure["error"] = pack(error)
+  failure["description"] = pack(description)
+  if not isChalkingOp() or errObject.isNone():
+    failedKeys[key] = pack(failure)
+  else:
+    errObject.get().failedKeys[key] = pack(failure)
+
 proc lookupByPath*(obj: ChalkDict, path: string): Option[Box] =
   let
     parts    = path.split(".")

tests/functional/imds/app.py

@@ -4,7 +4,7 @@
 # (see https://crashoverride.com/docs/chalk)
 import json
 
-from fastapi import FastAPI
+from fastapi import FastAPI, HTTPException, Request
 from fastapi.responses import PlainTextResponse
 
 
@@ -622,7 +622,9 @@ def health():
 
 
 @app.put("/latest/api/token", response_class=PlainTextResponse)
-def token():
+def token(request: Request):
+    if request.url.hostname != "169.254.169.254":
+        raise HTTPException(status_code=403)
     return TOKEN
 
 

tests/functional/test_plugins.py

@@ -158,7 +158,7 @@ def test_aws_no_imds(
             config=CONFIGS / "imds.c4m",
             env={
                 "VENDOR": str(vendor),
-                "METADATA_IP": "127.0.0.1",
+                "METADATA_IP": "imds",
                 "INSTANCE": str(instance),
             },
         )
@@ -167,6 +167,11 @@ def test_aws_no_imds(
                 "_OP_CLOUD_SYS_VENDOR": "Amazon EC2",
                 "_OP_CLOUD_PROVIDER": "aws",
                 "_OP_CLOUD_PROVIDER_SERVICE_TYPE": "aws_ec2",
+                "_OP_FAILED_KEYS": {
+                    "_OP_CLOUD_METADATA": {
+                        "code": "IMDS_DISABLED",
+                    },
+                },
                 "_AWS_INSTANCE_ID": "i-abc123xyz789",
             }
         )

tests/functional/testing.c4m

@@ -9,6 +9,9 @@ report_template insertion_default {
   key._OP_EXIT_CODE.use = true
 }
 
+report_template report_default {
+}
+
 if not env_exists("CHALK_USAGE_URL") {
   crashoverride_usage_reporting_url: "https://chalk-test.crashoverride.run/v0.1/usage"
 }