From 8ce1638eb07286e8e920b67fd63f5977c29c3df7 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:07:39 +1100 Subject: [PATCH] =?UTF-8?q?[8.x]=20=F0=9F=8C=8A=20Add=20failure=20store=20?= =?UTF-8?q?and=20exists=20conditions=20(#200861)=20(#202614)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Backport This will backport the following commits from `main` to `8.x`: - [🌊 Add failure store and exists conditions (#200861)](https://github.com/elastic/kibana/pull/200861) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Joe Reuter --- x-pack/plugins/streams/common/types.ts | 14 +++++++++++- .../component_templates/generate_layer.ts | 8 ++++++- .../helpers/condition_to_painless.test.ts | 8 +++++++ .../streams/helpers/condition_to_painless.ts | 22 +++++++++++++++++-- .../generate_index_template.ts | 1 + 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index d3aa43911ec2c..82a7a372662e8 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -9,13 +9,25 @@ import { z } from '@kbn/zod'; const stringOrNumberOrBoolean = z.union([z.string(), z.number(), z.boolean()]); -export const filterConditionSchema = z.object({ +export const binaryConditionSchema = z.object({ field: z.string(), operator: z.enum(['eq', 'neq', 'lt', 'lte', 'gt', 'gte', 'contains', 'startsWith', 'endsWith']), value: stringOrNumberOrBoolean, }); +export const unaryFilterConditionSchema = z.object({ + field: z.string(), + operator: z.enum(['exists', 'notExists']), +}); + +export const filterConditionSchema = z.discriminatedUnion('operator', [ + unaryFilterConditionSchema, + binaryConditionSchema, +]); + export type FilterCondition = z.infer; +export type BinaryFilterCondition = z.infer; +export type UnaryFilterCondition = z.infer; export interface AndCondition { and: Condition[]; diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index 4763aacb44478..69da4c5d8287d 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -7,6 +7,7 @@ import { ClusterPutComponentTemplateRequest, + MappingDateProperty, MappingProperty, } from '@elastic/elasticsearch/lib/api/types'; import { StreamDefinition } from '../../../../common/types'; @@ -21,9 +22,14 @@ export function generateLayer( ): ClusterPutComponentTemplateRequest { const properties: Record = {}; definition.fields.forEach((field) => { - properties[field.name] = { + const property: MappingProperty = { type: field.type, }; + if (field.name === '@timestamp') { + // @timestamp can't ignore malformed dates as it's used for sorting in logsdb + (property as MappingDateProperty).ignore_malformed = false; + } + properties[field.name] = property; }); return { name: getComponentTemplateName(id), diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts index 8c63d7caa8811..db0a5eaea8efd 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts @@ -44,6 +44,14 @@ const operatorConditionAndResults = [ condition: { field: 'log.logger', operator: 'contains' as const, value: 'proxy' }, result: '(ctx.log?.logger !== null && ctx.log?.logger.contains("proxy"))', }, + { + condition: { field: 'log.logger', operator: 'exists' as const }, + result: 'ctx.log?.logger !== null', + }, + { + condition: { field: 'log.logger', operator: 'notExists' as const }, + result: 'ctx.log?.logger == null', + }, ]; describe('conditionToPainless', () => { diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts index 2cccef260d7e1..dccc15b2ec8fc 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts @@ -8,11 +8,13 @@ import { isBoolean, isString } from 'lodash'; import { AndCondition, + BinaryFilterCondition, Condition, conditionSchema, FilterCondition, filterConditionSchema, RerouteOrCondition, + UnaryFilterCondition, } from '../../../../common/types'; function isFilterCondition(subject: any): subject is FilterCondition { @@ -44,7 +46,7 @@ function encodeValue(value: string | number | boolean) { return value; } -function toPainless(condition: FilterCondition) { +function binaryToPainless(condition: BinaryFilterCondition) { switch (condition.operator) { case 'neq': return `${safePainlessField(condition)} != ${encodeValue(condition.value)}`; @@ -67,9 +69,25 @@ function toPainless(condition: FilterCondition) { } } +function unaryToPainless(condition: UnaryFilterCondition) { + switch (condition.operator) { + case 'notExists': + return `${safePainlessField(condition)} == null`; + default: + return `${safePainlessField(condition)} !== null`; + } +} + +function isUnaryFilterCondition(subject: FilterCondition): subject is UnaryFilterCondition { + return !('value' in subject); +} + export function conditionToPainless(condition: Condition, nested = false): string { if (isFilterCondition(condition)) { - return `(${safePainlessField(condition)} !== null && ${toPainless(condition)})`; + if (isUnaryFilterCondition(condition)) { + return unaryToPainless(condition); + } + return `(${safePainlessField(condition)} !== null && ${binaryToPainless(condition)})`; } if (isAndCondition(condition)) { const and = condition.and.map((filter) => conditionToPainless(filter, true)).join(' && '); diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts index 7a16534a618da..0134951c8079a 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -27,6 +27,7 @@ export function generateIndexTemplate(id: string) { }, data_stream: { hidden: false, + failure_store: true, }, template: { settings: {