From cf65f42679c6940ea5308701f816337b0df0cdd8 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Thu, 18 Mar 2021 10:07:39 +0100 Subject: [PATCH] [Ingest Pipelines] Fix serialization and deserialization of user input for "patterns" fields (#94689) * updated serialization and deserialization behavior of dissect and gsub processors, also addded a test * also fix grok processor * pivot input checking to use JSON.stringify and JSON.parse Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../drag_and_drop_text_list.tsx | 20 +++++-- .../processor_form/processors/dissect.tsx | 7 ++- .../processor_form/processors/grok.tsx | 14 +++-- .../processor_form/processors/gsub.tsx | 9 +++- .../processor_form/processors/shared.test.ts | 53 +++++++++++++++++++ .../processor_form/processors/shared.ts | 51 +++++++++++++++++- 6 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/drag_and_drop_text_list.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/drag_and_drop_text_list.tsx index abe4eb0fa591..03bdc2ceb957 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/drag_and_drop_text_list.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/field_components/drag_and_drop_text_list.tsx @@ -44,7 +44,15 @@ interface Props { /** * Validation to be applied to every text item */ - textValidation?: ValidationFunc; + textValidations?: Array>; + /** + * Serializer to be applied to every text item + */ + textSerializer?: (v: string) => O; + /** + * Deserializer to be applied to every text item + */ + textDeserializer?: (v: unknown) => string; } const i18nTexts = { @@ -63,7 +71,9 @@ function DragAndDropTextListComponent({ onAdd, onRemove, addLabel, - textValidation, + textValidations, + textDeserializer, + textSerializer, }: Props): JSX.Element { const [droppableId] = useState(() => uuid.v4()); const [firstItemId] = useState(() => uuid.v4()); @@ -133,9 +143,11 @@ function DragAndDropTextListComponent({ path={item.path} config={{ - validations: textValidation - ? [{ validator: textValidation }] + validations: textValidations + ? textValidations.map((validator) => ({ validator })) : undefined, + deserializer: textDeserializer, + serializer: textSerializer, }} readDefaultValueOnForm={!item.isNew} > diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/dissect.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/dissect.tsx index 6652ad277cc2..3864581317e3 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/dissect.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/dissect.tsx @@ -22,7 +22,7 @@ import { import { FieldNameField } from './common_fields/field_name_field'; import { IgnoreMissingField } from './common_fields/ignore_missing_field'; -import { EDITOR_PX_HEIGHT, from } from './shared'; +import { EDITOR_PX_HEIGHT, from, to, isJSONStringValidator } from './shared'; const { emptyField } = fieldValidators; @@ -34,6 +34,8 @@ const getFieldsConfig = (esDocUrl: string): Record => { label: i18n.translate('xpack.ingestPipelines.pipelineEditor.dissectForm.patternFieldLabel', { defaultMessage: 'Pattern', }), + deserializer: to.escapeBackslashes, + serializer: from.unescapeBackslashes, helpText: ( => { ) ), }, + { + validator: isJSONStringValidator, + }, ], }, /* Optional field config */ diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx index f15441ea1f92..ae2d341c58c3 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/grok.tsx @@ -5,6 +5,7 @@ * 2.0. */ +import { flow } from 'lodash'; import React, { FunctionComponent } from 'react'; import { i18n } from '@kbn/i18n'; @@ -22,7 +23,7 @@ import { XJsonEditor, DragAndDropTextList } from '../field_components'; import { FieldNameField } from './common_fields/field_name_field'; import { IgnoreMissingField } from './common_fields/ignore_missing_field'; -import { FieldsConfig, to, from, EDITOR_PX_HEIGHT } from './shared'; +import { FieldsConfig, to, from, EDITOR_PX_HEIGHT, isJSONStringValidator } from './shared'; const { isJsonField, emptyField } = fieldValidators; @@ -46,7 +47,10 @@ const patternsValidation: ValidationFunc = ({ value, f } }; -const patternValidation = emptyField(valueRequiredMessage); +const patternValidations: Array> = [ + emptyField(valueRequiredMessage), + isJSONStringValidator, +]; const fieldsConfig: FieldsConfig = { /* Required field configs */ @@ -54,6 +58,8 @@ const fieldsConfig: FieldsConfig = { label: i18n.translate('xpack.ingestPipelines.pipelineEditor.grokForm.patternsFieldLabel', { defaultMessage: 'Patterns', }), + deserializer: flow(String, to.escapeBackslashes), + serializer: from.unescapeBackslashes, helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.grokForm.patternsHelpText', { defaultMessage: 'Grok expressions used to match and extract named capture groups. Uses the first matching expression.', @@ -133,7 +139,9 @@ export const Grok: FunctionComponent = () => { onAdd={addItem} onRemove={removeItem} addLabel={i18nTexts.addPatternLabel} - textValidation={patternValidation} + textValidations={patternValidations} + textDeserializer={fieldsConfig.patterns?.deserializer} + textSerializer={fieldsConfig.patterns?.serializer} /> ); }} diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/gsub.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/gsub.tsx index edfa59ea8028..11d06f3cca6f 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/gsub.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/gsub.tsx @@ -5,6 +5,7 @@ * 2.0. */ +import { flow } from 'lodash'; import React, { FunctionComponent } from 'react'; import { i18n } from '@kbn/i18n'; @@ -12,7 +13,7 @@ import { FIELD_TYPES, fieldValidators, UseField, Field } from '../../../../../.. import { TextEditor } from '../field_components'; -import { EDITOR_PX_HEIGHT, FieldsConfig } from './shared'; +import { EDITOR_PX_HEIGHT, FieldsConfig, from, to, isJSONStringValidator } from './shared'; import { FieldNameField } from './common_fields/field_name_field'; import { IgnoreMissingField } from './common_fields/ignore_missing_field'; import { TargetField } from './common_fields/target_field'; @@ -26,7 +27,8 @@ const fieldsConfig: FieldsConfig = { label: i18n.translate('xpack.ingestPipelines.pipelineEditor.gsubForm.patternFieldLabel', { defaultMessage: 'Pattern', }), - deserializer: String, + deserializer: flow(String, to.escapeBackslashes), + serializer: from.unescapeBackslashes, helpText: i18n.translate('xpack.ingestPipelines.pipelineEditor.gsubForm.patternFieldHelpText', { defaultMessage: 'Regular expression used to match substrings in the field.', }), @@ -38,6 +40,9 @@ const fieldsConfig: FieldsConfig = { }) ), }, + { + validator: isJSONStringValidator, + }, ], }, diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts new file mode 100644 index 000000000000..4b01f22a9383 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.test.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { from, to } from './shared'; + +describe('shared', () => { + describe('deserialization helpers', () => { + // This is the text that will be passed to the text input + test('to.escapeBackslashes', () => { + // this input loaded from the server + const input1 = 'my\ttab'; + expect(to.escapeBackslashes(input1)).toBe('my\\ttab'); + + // this input loaded from the server + const input2 = 'my\\ttab'; + expect(to.escapeBackslashes(input2)).toBe('my\\\\ttab'); + + // this input loaded from the server + const input3 = '\t\n\rOK'; + expect(to.escapeBackslashes(input3)).toBe('\\t\\n\\rOK'); + + const input4 = `%{clientip} %{ident} %{auth} [%{@timestamp}] \"%{verb} %{request} HTTP/%{httpversion}\" %{status} %{size}`; + expect(to.escapeBackslashes(input4)).toBe( + '%{clientip} %{ident} %{auth} [%{@timestamp}] \\"%{verb} %{request} HTTP/%{httpversion}\\" %{status} %{size}' + ); + }); + }); + + describe('serialization helpers', () => { + test('from.unescapeBackslashes', () => { + // user typed in "my\ttab" + const input1 = 'my\\ttab'; + expect(from.unescapeBackslashes(input1)).toBe('my\ttab'); + + // user typed in "my\\tab" + const input2 = 'my\\\\ttab'; + expect(from.unescapeBackslashes(input2)).toBe('my\\ttab'); + + // user typed in "\t\n\rOK" + const input3 = '\\t\\n\\rOK'; + expect(from.unescapeBackslashes(input3)).toBe('\t\n\rOK'); + + const input5 = `%{clientip} %{ident} %{auth} [%{@timestamp}] \\"%{verb} %{request} HTTP/%{httpversion}\\" %{status} %{size}`; + expect(from.unescapeBackslashes(input5)).toBe( + `%{clientip} %{ident} %{auth} [%{@timestamp}] \"%{verb} %{request} HTTP/%{httpversion}\" %{status} %{size}` + ); + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts index 399da3c05c78..bafba412c767 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/shared.ts @@ -5,11 +5,12 @@ * 2.0. */ -import { FunctionComponent } from 'react'; +import type { FunctionComponent } from 'react'; import * as rt from 'io-ts'; +import { i18n } from '@kbn/i18n'; import { isRight } from 'fp-ts/lib/Either'; -import { FieldConfig } from '../../../../../../shared_imports'; +import { FieldConfig, ValidationFunc } from '../../../../../../shared_imports'; export const arrayOfStrings = rt.array(rt.string); @@ -36,6 +37,17 @@ export const to = { arrayOfStrings: (v: unknown): string[] => isArrayOfStrings(v) ? v : typeof v === 'string' && v.length ? [v] : [], jsonString: (v: unknown) => (v ? JSON.stringify(v, null, 2) : '{}'), + /** + * Useful when deserializing strings that will be rendered inside of text areas or text inputs. We want + * a string like: "my\ttab" to render the same, not to render as "mytab". + */ + escapeBackslashes: (v: unknown) => { + if (typeof v === 'string') { + const s = JSON.stringify(v); + return s.slice(1, s.length - 1); + } + return v; + }, }; /** @@ -69,6 +81,41 @@ export const from = { optionalArrayOfStrings: (v: string[]) => (v.length ? v : undefined), undefinedIfValue: (value: unknown) => (v: boolean) => (v === value ? undefined : v), emptyStringToUndefined: (v: unknown) => (v === '' ? undefined : v), + /** + * Useful when serializing user input from a