[ResponseOps] Write rule execution results to Event Log

[banderror]

Related to: #135127

Summary

When #135127 is implemented and Security Solution rules start passing execution outcomes and custom metrics to the Alerting Framework, we would like the Framework to start writing "rule execution result" events to Event Log.

We could then query these events in Security Solution to:

• show them in the UI on the Rule Details page in the form of logs
• build rule monitoring analytics on top of them

Example

Here's an example of what this rule execution result event might look like:

{
  "@timestamp": "2022-06-22T20:23:58.584Z",
  "event": {
    "provider": "alerting",
    "kind": "event",
    "action": "execution-result",
    "sequence": 16,
    "severity": 10 // corresponds to log.level=debug
  },
  "rule": {
    "id": "b1343390-eb1d-11ec-929a-b1e8f41ac6d1", // rule id
    "uuid": "2c66bf23-6ae9-4eb2-859e-446bea181ae1", // rule.params.rule_id
    "name": "Rule with Related Integrations",
    "category": "siem.queryRule"
  },
  "log": {
    "level": "debug"
  },
  "kibana": {
    "alert": {
      "rule": {
        "rule_type_id": "siem.queryRule",
        "consumer": "siem",
        "execution": {
          "uuid": "656f296d-76e4-4da1-99e9-3601d2400ce9",
          // Outcome passed from Security Solution executor:
          "outcome": "10-succeeded", // 10-succeeded | 20-warning | 30-failed
          // Outcome message passed from Security Solution executor:
          "outcome_msg": "Rule executed successfully",
          // Warnings passed from Security Solution executor:
          "warning": ["max alerts limit hit"],
          "metrics": {
            // Custom metrics passed from Security Solution executor:
            "total_search_duration_ms": 1234567890,
            "total_indexing_duration_ms": 123456789,
            "total_alerts_detected": 543,
            "total_alerts_created": 100,
            "gap_duration_s": 7,
            // Framework's own internal metrics:
            "duration": 1234567890,
            // etc
          }
        }
      }
    },
    "space_ids": [
      "default"
    ],
    "saved_objects": [
      {
        "rel": "primary",
        "type": "alert",
        "id": "b1343390-eb1d-11ec-929a-b1e8f41ac6d1"
      }
    ],
    "server_uuid": "5b2de169-2785-441b-ae8c-186a1936b17d",
    "version": "8.4.0"
  },
  "ecs": {
    "version": "1.8.0"
  }
}

Example of Security Solution events

There are two types of events that we currently write to Event Log on the Security Solution side that we would get rid of when this issue is addressed: status-change and execution-metrics.

GET .kibana-event-log-*/_search
{
  "size": 1,
  "query": {
    "bool": {
      "filter": [
        {
          "term": {
            "event.provider": "securitySolution.ruleExecution"
          }
        },
        {
          "term": {
            "event.action": "status-change"
          }
        }
      ]
    }
  },
  "sort": [
    { "@timestamp": { "order": "desc" } },
    { "event.sequence": { "order": "desc" } }
  ]
}

{
  "took": 0,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 566,
      "relation": "eq"
    },
    "max_score": null,
    "hits": [
      {
        "_index": ".kibana-event-log-8.4.0-000001",
        "_id": "mBAWjYEBTgalB3qmAXKx",
        "_score": null,
        "_source": {
          "@timestamp": "2022-06-22T20:23:58.584Z",
          "event": {
            "provider": "securitySolution.ruleExecution",
            "kind": "event",
            "action": "status-change",
            "sequence": 17,
            "severity": 20
          },
          "message": "Rule execution completed successfully",
          "rule": {
            "id": "b1343390-eb1d-11ec-929a-b1e8f41ac6d1",
            "uuid": "2c66bf23-6ae9-4eb2-859e-446bea181ae1",
            "name": "Rule with Related Integrations",
            "category": "siem.queryRule"
          },
          "log": {
            "level": "info"
          },
          "kibana": {
            "alert": {
              "rule": {
                "execution": {
                  "uuid": "656f296d-76e4-4da1-99e9-3601d2400ce9",
                  "status": "succeeded",
                  "status_order": 0
                }
              }
            },
            "space_ids": [
              "default"
            ],
            "saved_objects": [
              {
                "rel": "primary",
                "type": "alert",
                "id": "b1343390-eb1d-11ec-929a-b1e8f41ac6d1"
              }
            ],
            "server_uuid": "5b2de169-2785-441b-ae8c-186a1936b17d",
            "version": "8.4.0"
          },
          "ecs": {
            "version": "1.8.0"
          }
        },
        "sort": [
          1655929438584,
          17
        ]
      }
    ]
  }
}

GET .kibana-event-log-*/_search
{
  "size": 1,
  "query": {
    "bool": {
      "filter": [
        {
          "term": {
            "event.provider": "securitySolution.ruleExecution"
          }
        },
        {
          "term": {
            "event.action": "execution-metrics"
          }
        }
      ]
    }
  },
  "sort": [
    { "@timestamp": { "order": "desc" } },
    { "event.sequence": { "order": "desc" } }
  ]
}

{
  "took": 0,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 283,
      "relation": "eq"
    },
    "max_score": null,
    "hits": [
      {
        "_index": ".kibana-event-log-8.4.0-000001",
        "_id": "lxAWjYEBTgalB3qmAXKx",
        "_score": null,
        "_source": {
          "@timestamp": "2022-06-22T20:23:58.584Z",
          "event": {
            "provider": "securitySolution.ruleExecution",
            "kind": "metric",
            "action": "execution-metrics",
            "sequence": 16,
            "severity": 10
          },
          "rule": {
            "id": "b1343390-eb1d-11ec-929a-b1e8f41ac6d1",
            "uuid": "2c66bf23-6ae9-4eb2-859e-446bea181ae1",
            "name": "Rule with Related Integrations",
            "category": "siem.queryRule"
          },
          "log": {
            "level": "debug"
          },
          "kibana": {
            "alert": {
              "rule": {
                "execution": {
                  "uuid": "656f296d-76e4-4da1-99e9-3601d2400ce9",
                  "metrics": {
                    "total_search_duration_ms": 4,
                    "total_indexing_duration_ms": 0
                  }
                }
              }
            },
            "space_ids": [
              "default"
            ],
            "saved_objects": [
              {
                "rel": "primary",
                "type": "alert",
                "id": "b1343390-eb1d-11ec-929a-b1e8f41ac6d1"
              }
            ],
            "server_uuid": "5b2de169-2785-441b-ae8c-186a1936b17d",
            "version": "8.4.0"
          },
          "ecs": {
            "version": "1.8.0"
          }
        },
        "sort": [
          1655929438584,
          16
        ]
      }
    ]
  }
}

[elasticmachine]

Pinging @elastic/security-detections-response (Team:Detections and Resp)

[elasticmachine]

Pinging @elastic/response-ops (Team:ResponseOps)

[elasticmachine]

Pinging @elastic/security-solution (Team: SecuritySolution)