Skip to content

Commit

Permalink
[Response Ops][Alerting] Update framework alerts client to write flat…
Browse files Browse the repository at this point in the history 
…tened alerts docs (#167439)

Resolves #166946

## Summary

The rule registry has traditionally written out AAD docs with flattened
keys, like

```
{
  "kibana.alert.rule.name": "test"
}
```

The framework alerts client has been writing out AAD docs as objects,
like

```
{
  "kibana": {
    "alert": {
      "rule": {
        "name": "test"
      }
    }
  }
}
```

We've identified a few places where we're updating the docs where having
this divergence makes things more difficult, so this is to switch the
framework to writing flattened alert docs before onboarding more rule
types.

This PR is targeted for 8.11, which is also when we onboarded the index
threshold rule type to FAAD. The only other rule type using FAAD to
write docs is ES query, which landed in 8.10 so there will be a followup
issue to handle the case of updating unflattened ES query AAD docs from
8.10

## To Verify

### ES Query and Index Threshold AaD

Create these rules that trigger alerts and verify that their AaD docs
are written out as flattened. For the ES Query rule type, select a
Metrics/Logs consumer and verify that they appear on the O11y alerts
table.

### ML alerts

ML alerts added in #166349 looked
like:

<details>
  <summary>Unflattened</summary>

```
{
	"kibana": {
		"alert": {
			"url": "/app/ml/explorer/?_g=(ml%3A(jobIds%3A!(rt-anomaly-mean-value))%2Ctime%3A(from%3A'2023-09-28T14%3A57%3A00.000Z'%2Cmode%3Aabsolute%2Cto%3A'2023-09-28T15%3A17%3A00.000Z'))&_a=(explorer%3A(mlExplorerFilter%3A(filterActive%3A!t%2CfilteredFields%3A!(key%2Cthird-key)%2CinfluencersFilterQuery%3A(bool%3A(minimum_should_match%3A1%2Cshould%3A!((match_phrase%3A(key%3Athird-key)))))%2CqueryString%3A'key%3A%22third-key%22')%2CmlExplorerSwimlane%3A()))",
			"reason": "Alerts are raised based on real-time scores. Remember that scores may be adjusted over time as data continues to be analyzed.",
			"job_id": "rt-anomaly-mean-value",
			"anomaly_score": 73.63508175828011,
			"is_interim": false,
			"anomaly_timestamp": 1695913620000,
			"top_records": [{
				"job_id": "rt-anomaly-mean-value",
				"record_score": 73.63516446528412,
				"initial_record_score": 73.63516446528412,
				"detector_index": 0,
				"is_interim": false,
				"timestamp": 1695913620000,
				"partition_field_name": "key",
				"partition_field_value": "third-key",
				"function": "mean",
				"actual": [
					3
				],
				"typical": [
					4.187715468532429
				]
			}],
			"top_influencers": [{
				"job_id": "rt-anomaly-mean-value",
				"influencer_field_name": "key",
				"influencer_field_value": "third-key",
				"influencer_score": 73.63508175828011,
				"initial_influencer_score": 73.63508175828011,
				"is_interim": false,
				"timestamp": 1695913620000
			}],
			"action_group": "anomaly_score_match",
			"flapping": false,
			"flapping_history": [
				true,
				false,
				false,
				false
			],
			"instance": {
				"id": "rt-anomaly-mean-value"
			},
			"maintenance_window_ids": [],
			"rule": {
				"category": "Anomaly detection alert",
				"consumer": "alerts",
				"execution": {
					"uuid": "e9e681d4-c8e4-43eb-82e5-a58bdf7ffe12"
				},
				"name": "rt-ad-alert-influencer",
				"parameters": {
					"severity": 5,
					"resultType": "influencer",
					"includeInterim": false,
					"jobSelection": {
						"jobIds": [
							"rt-anomaly-mean-value"
						],
						"groupIds": []
					},
					"lookbackInterval": null,
					"topNBuckets": null
				},
				"producer": "ml",
				"revision": 0,
				"rule_type_id": "xpack.ml.anomaly_detection_alert",
				"tags": [],
				"uuid": "9e1d6bc0-5e10-11ee-8416-3bf48cca0922"
			},
			"status": "active",
			"uuid": "c9c1f075-9985-4c55-8ff8-22349cb30269",
			"workflow_status": "open",
			"duration": {
				"us": "99021000000"
			},
			"start": "2023-09-28T15:07:12.868Z",
			"time_range": {
				"gte": "2023-09-28T15:07:12.868Z"
			}
		},
		"space_ids": [
			"default"
		],
		"version": "8.11.0"
	},
	"@timestamp": "2023-09-28T15:08:51.889Z",
	"event": {
		"action": "active",
		"kind": "signal"
	},
	"tags": []
}
```
</details>

Now they look like:

<details>
  <summary>Flattened</summary>

```
{
	"kibana.alert.url": "/app/ml/explorer/?_g=(ml%3A(jobIds%3A!(rt-anomaly-mean-value))%2Ctime%3A(from%3A'2023-09-28T15%3A03%3A00.000Z'%2Cmode%3Aabsolute%2Cto%3A'2023-09-28T15%3A23%3A00.000Z'))&_a=(explorer%3A(mlExplorerFilter%3A(filterActive%3A!t%2CfilteredFields%3A!(key%2Cthird-key)%2CinfluencersFilterQuery%3A(bool%3A(minimum_should_match%3A1%2Cshould%3A!((match_phrase%3A(key%3Athird-key)))))%2CqueryString%3A'key%3A%22third-key%22')%2CmlExplorerSwimlane%3A()))",
	"kibana.alert.reason": "Alerts are raised based on real-time scores. Remember that scores may be adjusted over time as data continues to be analyzed.",
	"kibana.alert.job_id": "rt-anomaly-mean-value",
	"kibana.alert.anomaly_score": 72.75515452061356,
	"kibana.alert.is_interim": false,
	"kibana.alert.anomaly_timestamp": 1695913980000,
	"kibana.alert.top_records": [{
		"job_id": "rt-anomaly-mean-value",
		"record_score": 72.75515452061356,
		"initial_record_score": 72.75515452061356,
		"detector_index": 0,
		"is_interim": false,
		"timestamp": 1695913980000,
		"partition_field_name": "key",
		"partition_field_value": "third-key",
		"function": "mean",
		"actual": [
			0.5
		],
		"typical": [
			4.138745343296527
		]
	}],
	"kibana.alert.top_influencers": [{
		"job_id": "rt-anomaly-mean-value",
		"influencer_field_name": "key",
		"influencer_field_value": "third-key",
		"influencer_score": 72.75515452061356,
		"initial_influencer_score": 72.75515452061356,
		"is_interim": false,
		"timestamp": 1695913980000
	}],
	"kibana.alert.rule.category": "Anomaly detection alert",
	"kibana.alert.rule.consumer": "alerts",
	"kibana.alert.rule.execution.uuid": "17fef3d3-d595-4362-837e-b2a73650169e",
	"kibana.alert.rule.name": "rt-ad-alert-influencer",
	"kibana.alert.rule.parameters": {
		"severity": 5,
		"resultType": "influencer",
		"includeInterim": false,
		"jobSelection": {
			"jobIds": [
				"rt-anomaly-mean-value"
			],
			"groupIds": []
		},
		"lookbackInterval": null,
		"topNBuckets": null
	},
	"kibana.alert.rule.producer": "ml",
	"kibana.alert.rule.revision": 0,
	"kibana.alert.rule.rule_type_id": "xpack.ml.anomaly_detection_alert",
	"kibana.alert.rule.tags": [],
	"kibana.alert.rule.uuid": "757c7610-5e11-11ee-8bc6-a95c3ced4757",
	"kibana.space_ids": [
		"default"
	],
	"@timestamp": "2023-09-28T15:14:52.057Z",
	"event.action": "active",
	"event.kind": "signal",
	"kibana.alert.action_group": "anomaly_score_match",
	"kibana.alert.flapping": false,
	"kibana.alert.flapping_history": [
		true,
		false,
		false,
		false
	],
	"kibana.alert.instance.id": "rt-anomaly-mean-value",
	"kibana.alert.maintenance_window_ids": [],
	"kibana.alert.status": "active",
	"kibana.alert.uuid": "ac1f0d7c-461b-4fc6-b4c3-04416ac876d3",
	"kibana.alert.workflow_status": "open",
	"kibana.alert.duration.us": "99028000000",
	"kibana.alert.start": "2023-09-28T15:13:13.028Z",
	"kibana.alert.time_range": {
		"gte": "2023-09-28T15:13:13.028Z"
	},
	"kibana.version": "8.11.0",
	"tags": []
}
```
</details>
  • Loading branch information
@ymao1
ymao1 authored Sep 29, 2023
1 parent 6431d7c commit 8dec3f6
Show file tree
Hide file tree
Showing 46 changed files with 3,632 additions and 4,832 deletions.
2 changes: 1 addition & 1 deletion packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export const createSchemaFromFieldMap = ({
useAlert = false,
useEcs = false,
useLegacyAlerts = false,
flattened = false,
flattened = true,
}: CreateSchemaFromFieldMapOpts) => {
const lineWriters = {
IMPORTS: createLineWriter(),
Expand Down
82 changes: 31 additions & 51 deletions packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,59 +68,39 @@ export const schemaGeoPointArray = rt.array(schemaGeoPoint);
// prettier-ignore
const AlertRequired = rt.type({
'@timestamp': schemaDate,
kibana: rt.type({
alert: rt.type({
instance: rt.type({
id: schemaString,
}),
rule: rt.type({
category: schemaString,
consumer: schemaString,
name: schemaString,
producer: schemaString,
revision: schemaStringOrNumber,
rule_type_id: schemaString,
uuid: schemaString,
}),
status: schemaString,
uuid: schemaString,
}),
space_ids: schemaStringArray,
}),
'kibana.alert.instance.id': schemaString,
'kibana.alert.rule.category': schemaString,
'kibana.alert.rule.consumer': schemaString,
'kibana.alert.rule.name': schemaString,
'kibana.alert.rule.producer': schemaString,
'kibana.alert.rule.revision': schemaStringOrNumber,
'kibana.alert.rule.rule_type_id': schemaString,
'kibana.alert.rule.uuid': schemaString,
'kibana.alert.status': schemaString,
'kibana.alert.uuid': schemaString,
'kibana.space_ids': schemaStringArray,
});
const AlertOptional = rt.partial({
event: rt.partial({
action: schemaString,
kind: schemaString,
}),
kibana: rt.partial({
alert: rt.partial({
action_group: schemaString,
case_ids: schemaStringArray,
duration: rt.partial({
us: schemaStringOrNumber,
}),
end: schemaDate,
flapping: schemaBoolean,
flapping_history: schemaBooleanArray,
last_detected: schemaDate,
maintenance_window_ids: schemaStringArray,
reason: schemaString,
rule: rt.partial({
execution: rt.partial({
uuid: schemaString,
}),
parameters: schemaUnknown,
tags: schemaStringArray,
}),
start: schemaDate,
time_range: schemaDateRange,
url: schemaString,
workflow_status: schemaString,
workflow_tags: schemaStringArray,
}),
version: schemaString,
}),
'event.action': schemaString,
'event.kind': schemaString,
'kibana.alert.action_group': schemaString,
'kibana.alert.case_ids': schemaStringArray,
'kibana.alert.duration.us': schemaStringOrNumber,
'kibana.alert.end': schemaDate,
'kibana.alert.flapping': schemaBoolean,
'kibana.alert.flapping_history': schemaBooleanArray,
'kibana.alert.last_detected': schemaDate,
'kibana.alert.maintenance_window_ids': schemaStringArray,
'kibana.alert.reason': schemaString,
'kibana.alert.rule.execution.uuid': schemaString,
'kibana.alert.rule.parameters': schemaUnknown,
'kibana.alert.rule.tags': schemaStringArray,
'kibana.alert.start': schemaDate,
'kibana.alert.time_range': schemaDateRange,
'kibana.alert.url': schemaString,
'kibana.alert.workflow_status': schemaString,
'kibana.alert.workflow_tags': schemaStringArray,
'kibana.version': schemaString,
tags: schemaStringArray,
});

Expand Down
Loading

0 comments on commit 8dec3f6

Please sign in to comment.