Skip to content

Commit

Permalink
[Response Ops][Alerting] Update framework alerts client to write flat…
Browse files Browse the repository at this point in the history 
…tened alerts docs (elastic#167691)

Resolves elastic#166946

## PRs to this feature branch
* elastic#167439
* elastic#167583

## Summary

The rule registry has traditionally written out AAD docs with flattened
keys, like

```
{
  "kibana.alert.rule.name": "test"
}
```

The framework alerts client has been writing out AAD docs as objects,
like

```
{
  "kibana": {
    "alert": {
      "rule": {
        "name": "test"
      }
    }
  }
}
```

We've identified a few places where we're updating the docs where having
this divergence makes things more difficult, so this is to switch the
framework to writing flattened alert docs before onboarding more rule
types.

This PR is targeted for 8.11, which is also when we onboarded the index
threshold rule type and the ML anomaly detection rule type to FAAD. For
the ES query rule, which started writing unflattened AaD docs in 8.10,
this PR adds special handling to ensure that those unflattened docs are
correctly updated with flattened fields.

## To Verify

### ES Query and Index Threshold AaD

Create these rules that trigger alerts and verify that their AaD docs
are written out as flattened. For the ES Query rule type, select a
Metrics/Logs consumer and verify that they appear on the O11y alerts
table.

### ML alerts

ML alerts added in elastic#166349 looked
like:

<details>
  <summary>Unflattened</summary>

```
{
	"kibana": {
		"alert": {
			"url": "/app/ml/explorer/?_g=(ml%3A(jobIds%3A!(rt-anomaly-mean-value))%2Ctime%3A(from%3A'2023-09-28T14%3A57%3A00.000Z'%2Cmode%3Aabsolute%2Cto%3A'2023-09-28T15%3A17%3A00.000Z'))&_a=(explorer%3A(mlExplorerFilter%3A(filterActive%3A!t%2CfilteredFields%3A!(key%2Cthird-key)%2CinfluencersFilterQuery%3A(bool%3A(minimum_should_match%3A1%2Cshould%3A!((match_phrase%3A(key%3Athird-key)))))%2CqueryString%3A'key%3A%22third-key%22')%2CmlExplorerSwimlane%3A()))",
			"reason": "Alerts are raised based on real-time scores. Remember that scores may be adjusted over time as data continues to be analyzed.",
			"job_id": "rt-anomaly-mean-value",
			"anomaly_score": 73.63508175828011,
			"is_interim": false,
			"anomaly_timestamp": 1695913620000,
			"top_records": [{
				"job_id": "rt-anomaly-mean-value",
				"record_score": 73.63516446528412,
				"initial_record_score": 73.63516446528412,
				"detector_index": 0,
				"is_interim": false,
				"timestamp": 1695913620000,
				"partition_field_name": "key",
				"partition_field_value": "third-key",
				"function": "mean",
				"actual": [
					3
				],
				"typical": [
					4.187715468532429
				]
			}],
			"top_influencers": [{
				"job_id": "rt-anomaly-mean-value",
				"influencer_field_name": "key",
				"influencer_field_value": "third-key",
				"influencer_score": 73.63508175828011,
				"initial_influencer_score": 73.63508175828011,
				"is_interim": false,
				"timestamp": 1695913620000
			}],
			"action_group": "anomaly_score_match",
			"flapping": false,
			"flapping_history": [
				true,
				false,
				false,
				false
			],
			"instance": {
				"id": "rt-anomaly-mean-value"
			},
			"maintenance_window_ids": [],
			"rule": {
				"category": "Anomaly detection alert",
				"consumer": "alerts",
				"execution": {
					"uuid": "e9e681d4-c8e4-43eb-82e5-a58bdf7ffe12"
				},
				"name": "rt-ad-alert-influencer",
				"parameters": {
					"severity": 5,
					"resultType": "influencer",
					"includeInterim": false,
					"jobSelection": {
						"jobIds": [
							"rt-anomaly-mean-value"
						],
						"groupIds": []
					},
					"lookbackInterval": null,
					"topNBuckets": null
				},
				"producer": "ml",
				"revision": 0,
				"rule_type_id": "xpack.ml.anomaly_detection_alert",
				"tags": [],
				"uuid": "9e1d6bc0-5e10-11ee-8416-3bf48cca0922"
			},
			"status": "active",
			"uuid": "c9c1f075-9985-4c55-8ff8-22349cb30269",
			"workflow_status": "open",
			"duration": {
				"us": "99021000000"
			},
			"start": "2023-09-28T15:07:12.868Z",
			"time_range": {
				"gte": "2023-09-28T15:07:12.868Z"
			}
		},
		"space_ids": [
			"default"
		],
		"version": "8.11.0"
	},
	"@timestamp": "2023-09-28T15:08:51.889Z",
	"event": {
		"action": "active",
		"kind": "signal"
	},
	"tags": []
}
```
</details>

Now they look like:

<details>
  <summary>Flattened</summary>

```
{
	"kibana.alert.url": "/app/ml/explorer/?_g=(ml%3A(jobIds%3A!(rt-anomaly-mean-value))%2Ctime%3A(from%3A'2023-09-28T15%3A03%3A00.000Z'%2Cmode%3Aabsolute%2Cto%3A'2023-09-28T15%3A23%3A00.000Z'))&_a=(explorer%3A(mlExplorerFilter%3A(filterActive%3A!t%2CfilteredFields%3A!(key%2Cthird-key)%2CinfluencersFilterQuery%3A(bool%3A(minimum_should_match%3A1%2Cshould%3A!((match_phrase%3A(key%3Athird-key)))))%2CqueryString%3A'key%3A%22third-key%22')%2CmlExplorerSwimlane%3A()))",
	"kibana.alert.reason": "Alerts are raised based on real-time scores. Remember that scores may be adjusted over time as data continues to be analyzed.",
	"kibana.alert.job_id": "rt-anomaly-mean-value",
	"kibana.alert.anomaly_score": 72.75515452061356,
	"kibana.alert.is_interim": false,
	"kibana.alert.anomaly_timestamp": 1695913980000,
	"kibana.alert.top_records": [{
		"job_id": "rt-anomaly-mean-value",
		"record_score": 72.75515452061356,
		"initial_record_score": 72.75515452061356,
		"detector_index": 0,
		"is_interim": false,
		"timestamp": 1695913980000,
		"partition_field_name": "key",
		"partition_field_value": "third-key",
		"function": "mean",
		"actual": [
			0.5
		],
		"typical": [
			4.138745343296527
		]
	}],
	"kibana.alert.top_influencers": [{
		"job_id": "rt-anomaly-mean-value",
		"influencer_field_name": "key",
		"influencer_field_value": "third-key",
		"influencer_score": 72.75515452061356,
		"initial_influencer_score": 72.75515452061356,
		"is_interim": false,
		"timestamp": 1695913980000
	}],
	"kibana.alert.rule.category": "Anomaly detection alert",
	"kibana.alert.rule.consumer": "alerts",
	"kibana.alert.rule.execution.uuid": "17fef3d3-d595-4362-837e-b2a73650169e",
	"kibana.alert.rule.name": "rt-ad-alert-influencer",
	"kibana.alert.rule.parameters": {
		"severity": 5,
		"resultType": "influencer",
		"includeInterim": false,
		"jobSelection": {
			"jobIds": [
				"rt-anomaly-mean-value"
			],
			"groupIds": []
		},
		"lookbackInterval": null,
		"topNBuckets": null
	},
	"kibana.alert.rule.producer": "ml",
	"kibana.alert.rule.revision": 0,
	"kibana.alert.rule.rule_type_id": "xpack.ml.anomaly_detection_alert",
	"kibana.alert.rule.tags": [],
	"kibana.alert.rule.uuid": "757c7610-5e11-11ee-8bc6-a95c3ced4757",
	"kibana.space_ids": [
		"default"
	],
	"@timestamp": "2023-09-28T15:14:52.057Z",
	"event.action": "active",
	"event.kind": "signal",
	"kibana.alert.action_group": "anomaly_score_match",
	"kibana.alert.flapping": false,
	"kibana.alert.flapping_history": [
		true,
		false,
		false,
		false
	],
	"kibana.alert.instance.id": "rt-anomaly-mean-value",
	"kibana.alert.maintenance_window_ids": [],
	"kibana.alert.status": "active",
	"kibana.alert.uuid": "ac1f0d7c-461b-4fc6-b4c3-04416ac876d3",
	"kibana.alert.workflow_status": "open",
	"kibana.alert.duration.us": "99028000000",
	"kibana.alert.start": "2023-09-28T15:13:13.028Z",
	"kibana.alert.time_range": {
		"gte": "2023-09-28T15:13:13.028Z"
	},
	"kibana.version": "8.11.0",
	"tags": []
}
```
</details>

---------

Co-authored-by: kibanamachine <[email protected]>
  • Loading branch information
@ymao1 @kibanamachine
ymao1 and kibanamachine authored Oct 2, 2023
1 parent 9153483 commit 3bff7fb
Show file tree
Hide file tree
Showing 51 changed files with 5,051 additions and 5,670 deletions.
16 changes: 8 additions & 8 deletions packages/kbn-alerts-as-data-utils/src/schemas/create_schema_from_field_map.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export const createSchemaFromFieldMap = ({
useAlert = false,
useEcs = false,
useLegacyAlerts = false,
flattened = false,
flattened = true,
}: CreateSchemaFromFieldMapOpts) => {
const lineWriters = {
IMPORTS: createLineWriter(),
Expand Down Expand Up @@ -280,19 +280,19 @@ export const IsoDateString = new rt.Type<string, string, unknown>(
rt.identity
);
export type IsoDateStringC = typeof IsoDateString;
export const schemaDate = IsoDateString;
export const schemaDateArray = rt.array(IsoDateString);
export const schemaDateRange = rt.partial({
gte: schemaDate,
lte: schemaDate,
});
export const schemaDateRangeArray = rt.array(schemaDateRange);
export const schemaUnknown = rt.unknown;
export const schemaUnknownArray = rt.array(rt.unknown);
export const schemaString = rt.string;
export const schemaStringArray = rt.array(schemaString);
export const schemaNumber = rt.number;
export const schemaNumberArray = rt.array(schemaNumber);
export const schemaDate = rt.union([IsoDateString, schemaNumber]);
export const schemaDateArray = rt.array(schemaDate);
export const schemaDateRange = rt.partial({
gte: schemaDate,
lte: schemaDate,
});
export const schemaDateRangeArray = rt.array(schemaDateRange);
export const schemaStringOrNumber = rt.union([schemaString, schemaNumber]);
export const schemaStringOrNumberArray = rt.array(schemaStringOrNumber);
export const schemaBoolean = rt.boolean;
Expand Down
96 changes: 38 additions & 58 deletions packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,19 @@ export const IsoDateString = new rt.Type<string, string, unknown>(
rt.identity
);
export type IsoDateStringC = typeof IsoDateString;
export const schemaDate = IsoDateString;
export const schemaDateArray = rt.array(IsoDateString);
export const schemaDateRange = rt.partial({
gte: schemaDate,
lte: schemaDate,
});
export const schemaDateRangeArray = rt.array(schemaDateRange);
export const schemaUnknown = rt.unknown;
export const schemaUnknownArray = rt.array(rt.unknown);
export const schemaString = rt.string;
export const schemaStringArray = rt.array(schemaString);
export const schemaNumber = rt.number;
export const schemaNumberArray = rt.array(schemaNumber);
export const schemaDate = rt.union([IsoDateString, schemaNumber]);
export const schemaDateArray = rt.array(schemaDate);
export const schemaDateRange = rt.partial({
gte: schemaDate,
lte: schemaDate,
});
export const schemaDateRangeArray = rt.array(schemaDateRange);
export const schemaStringOrNumber = rt.union([schemaString, schemaNumber]);
export const schemaStringOrNumberArray = rt.array(schemaStringOrNumber);
export const schemaBoolean = rt.boolean;
Expand Down Expand Up @@ -68,59 +68,39 @@ export const schemaGeoPointArray = rt.array(schemaGeoPoint);
// prettier-ignore
const AlertRequired = rt.type({
'@timestamp': schemaDate,
kibana: rt.type({
alert: rt.type({
instance: rt.type({
id: schemaString,
}),
rule: rt.type({
category: schemaString,
consumer: schemaString,
name: schemaString,
producer: schemaString,
revision: schemaStringOrNumber,
rule_type_id: schemaString,
uuid: schemaString,
}),
status: schemaString,
uuid: schemaString,
}),
space_ids: schemaStringArray,
}),
'kibana.alert.instance.id': schemaString,
'kibana.alert.rule.category': schemaString,
'kibana.alert.rule.consumer': schemaString,
'kibana.alert.rule.name': schemaString,
'kibana.alert.rule.producer': schemaString,
'kibana.alert.rule.revision': schemaStringOrNumber,
'kibana.alert.rule.rule_type_id': schemaString,
'kibana.alert.rule.uuid': schemaString,
'kibana.alert.status': schemaString,
'kibana.alert.uuid': schemaString,
'kibana.space_ids': schemaStringArray,
});
const AlertOptional = rt.partial({
event: rt.partial({
action: schemaString,
kind: schemaString,
}),
kibana: rt.partial({
alert: rt.partial({
action_group: schemaString,
case_ids: schemaStringArray,
duration: rt.partial({
us: schemaStringOrNumber,
}),
end: schemaDate,
flapping: schemaBoolean,
flapping_history: schemaBooleanArray,
last_detected: schemaDate,
maintenance_window_ids: schemaStringArray,
reason: schemaString,
rule: rt.partial({
execution: rt.partial({
uuid: schemaString,
}),
parameters: schemaUnknown,
tags: schemaStringArray,
}),
start: schemaDate,
time_range: schemaDateRange,
url: schemaString,
workflow_status: schemaString,
workflow_tags: schemaStringArray,
}),
version: schemaString,
}),
'event.action': schemaString,
'event.kind': schemaString,
'kibana.alert.action_group': schemaString,
'kibana.alert.case_ids': schemaStringArray,
'kibana.alert.duration.us': schemaStringOrNumber,
'kibana.alert.end': schemaDate,
'kibana.alert.flapping': schemaBoolean,
'kibana.alert.flapping_history': schemaBooleanArray,
'kibana.alert.last_detected': schemaDate,
'kibana.alert.maintenance_window_ids': schemaStringArray,
'kibana.alert.reason': schemaString,
'kibana.alert.rule.execution.uuid': schemaString,
'kibana.alert.rule.parameters': schemaUnknown,
'kibana.alert.rule.tags': schemaStringArray,
'kibana.alert.start': schemaDate,
'kibana.alert.time_range': schemaDateRange,
'kibana.alert.url': schemaString,
'kibana.alert.workflow_status': schemaString,
'kibana.alert.workflow_tags': schemaStringArray,
'kibana.version': schemaString,
tags: schemaStringArray,
});

Expand Down
Loading

0 comments on commit 3bff7fb

Please sign in to comment.