From 9b8fff835f26e0c4994a8d1622601cb4bd292da4 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 20 Nov 2024 12:02:37 -0800 Subject: [PATCH] Set up the config -> form -> schema -> template transformations for input_map Signed-off-by: Tyler Ohlsen --- common/constants.ts | 6 + common/interfaces.ts | 26 +- public/configs/ml_processor.ts | 2 +- .../ml_processor_inputs.tsx | 17 +- .../ml_processor_inputs/model_inputs.tsx | 296 +++++++++++++++--- public/utils/config_to_form_utils.ts | 10 +- public/utils/config_to_schema_utils.ts | 23 ++ public/utils/config_to_template_utils.ts | 32 +- 8 files changed, 357 insertions(+), 55 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index 69826bc7..ef507f4d 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -506,3 +506,9 @@ export enum SOURCE_OPTIONS { UPLOAD = 'upload', EXISTING_INDEX = 'existing_index', } +export enum TRANSFORM_TYPE { + STRING = 'string', + FIELD = 'field', + EXPRESSION = 'expression', + TEMPLATE = 'template', +} diff --git a/common/interfaces.ts b/common/interfaces.ts index f9bda443..acd36db7 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -10,6 +10,7 @@ import { COMPONENT_CLASS, PROCESSOR_TYPE, PROMPT_FIELD, + TRANSFORM_TYPE, WORKFLOW_TYPE, } from './constants'; @@ -35,9 +36,21 @@ export type ConfigFieldType = | 'map' | 'mapArray' | 'boolean' - | 'number'; + | 'number' + | 'transform' + | 'transformArray'; -export type ConfigFieldValue = string | {}; +export type ConfigFieldTransformValue = { + transformType: TRANSFORM_TYPE; + value: string; +}; +export type ConfigFieldTransformArrayValue = ConfigFieldTransformValue[]; + +export type ConfigFieldValue = + | string + | ConfigFieldTransformValue + | ConfigFieldTransformArrayValue + | {}; export interface IConfigField { type: ConfigFieldType; @@ -100,6 +113,15 @@ export type MapFormValue = MapEntry[]; export type MapArrayFormValue = MapFormValue[]; +export type InputMapEntry = { + key: string; + value: ConfigFieldTransformValue; +}; + +export type InputMapFormValue = InputMapEntry[]; + +export type InputMapArrayFormValue = InputMapFormValue[]; + export type WorkflowFormValues = { ingest: FormikValues; search: FormikValues; diff --git a/public/configs/ml_processor.ts b/public/configs/ml_processor.ts index 623981ae..83254d2d 100644 --- a/public/configs/ml_processor.ts +++ b/public/configs/ml_processor.ts @@ -22,7 +22,7 @@ export abstract class MLProcessor extends Processor { }, { id: 'input_map', - type: 'mapArray', + type: 'transformArray', }, { id: 'output_map', diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx index 90c525e7..74476d93 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx @@ -33,6 +33,12 @@ import { MapFormValue, } from '../../../../../../common'; import { ModelField } from '../../input_fields'; +import { + TRANSFORM_TYPE, + InputMapEntry, + InputMapFormValue, + InputMapArrayFormValue, +} from '../../../../../../common'; import { ConfigurePromptModal, InputTransformModal, @@ -139,10 +145,13 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { parseModelInputs(newModelInterface).map((modelInput) => { return { key: modelInput.label, - value: '', - } as MapEntry; - }) as MapFormValue, - ] as MapArrayFormValue; + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: '', + }, + } as InputMapEntry; + }) as InputMapFormValue, + ] as InputMapArrayFormValue; const modelOutputsAsForm = [ parseModelOutputs(newModelInterface).map((modelOutput) => { return { diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx index 4d18aa44..cc789c71 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx @@ -4,7 +4,7 @@ */ import React, { useState, useEffect } from 'react'; -import { useFormikContext, getIn } from 'formik'; +import { useFormikContext, getIn, Field, FieldProps } from 'formik'; import { isEmpty } from 'lodash'; import { useSelector } from 'react-redux'; import { flattie } from 'flattie'; @@ -17,14 +17,29 @@ import { IndexMappings, REQUEST_PREFIX, REQUEST_PREFIX_WITH_JSONPATH_ROOT_SELECTOR, + InputMapEntry, + InputMapFormValue, + TRANSFORM_TYPE, } from '../../../../../../common'; -import { MapArrayField } from '../../input_fields'; +import { TextField } from '../../input_fields'; import { AppState, getMappings, useAppDispatch } from '../../../../../store'; import { getDataSourceId, parseModelInputs, sanitizeJSONPath, } from '../../../../../utils'; +import { + EuiCompressedFormRow, + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiIconTip, + EuiPanel, + EuiSmallButtonEmpty, + EuiSmallButtonIcon, + EuiText, +} from '@elastic/eui'; +import { SelectWithCustomOptions } from '../../input_fields/select_with_custom_options'; interface ModelInputsProps { config: IProcessorConfig; @@ -32,6 +47,11 @@ interface ModelInputsProps { context: PROCESSOR_CONTEXT; } +// The keys will be more static in general. Give more space for values where users +// will typically be writing out more complex transforms/configuration (in the case of ML inference processors). +const KEY_FLEX_RATIO = 4; +const VALUE_FLEX_RATIO = 6; + /** * Base component to configure ML inputs. */ @@ -40,14 +60,21 @@ export function ModelInputs(props: ModelInputsProps) { const dataSourceId = getDataSourceId(); const { models } = useSelector((state: AppState) => state.ml); const indices = useSelector((state: AppState) => state.opensearch.indices); - const { values } = useFormikContext(); - + const { + setFieldValue, + setFieldTouched, + errors, + touched, + values, + } = useFormikContext(); // get some current form & config values const modelField = props.config.fields.find( (field) => field.type === 'model' ) as IConfigField; const modelFieldPath = `${props.baseConfigPath}.${props.config.id}.${modelField.id}`; - const inputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.input_map`; + // Assuming no more than one set of input map entries. + // TODO: confirm the above. + const inputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.input_map.0`; // model interface state const [modelInterface, setModelInterface] = useState< @@ -141,45 +168,226 @@ export function ModelInputs(props: ModelInputsProps) { } }, [values?.search?.index?.name]); + // Adding a map entry to the end of the existing arr + function addMapEntry(curEntries: InputMapFormValue): void { + const updatedEntries = [ + ...curEntries, + { + key: '', + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: '', + }, + } as InputMapEntry, + ]; + setFieldValue(inputMapFieldPath, updatedEntries); + setFieldTouched(inputMapFieldPath, true); + } + + // Deleting a map entry + function deleteMapEntry( + curEntries: InputMapFormValue, + entryIndexToDelete: number + ): void { + const updatedEntries = [...curEntries]; + updatedEntries.splice(entryIndexToDelete, 1); + setFieldValue(inputMapFieldPath, updatedEntries); + setFieldTouched(inputMapFieldPath, true); + } + + // Defining constants for the key/value text vars, typically dependent on the different processor contexts. + const keyTitle = 'Name'; + const keyPlaceholder = 'Name'; + const keyOptions = parseModelInputs(modelInterface); + const valueTitle = + props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? 'Query field' + : 'Document field'; + const valuePlaceholder = + props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? 'Specify a query field' + : 'Define a document field'; + const valueHelpText = `Specify a ${ + props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST ? 'query' : 'document' + } field or define JSONPath to transform the ${ + props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST ? 'query' : 'document' + } to map to a model input field.${ + props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE + ? ` Or, if you'd like to include data from the the original query request, prefix your mapping with "${REQUEST_PREFIX}" or "${REQUEST_PREFIX_WITH_JSONPATH_ROOT_SELECTOR}" - for example, "_request.query.match.my_field"` + : '' + }`; + const valueOptions = + props.context === PROCESSOR_CONTEXT.INGEST + ? docFields + : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? queryFields + : indexMappingFields; + return ( - + + + {({ field, form }: FieldProps) => { + return ( + 0 + ? 'Invalid or missing mapping values' + : false + } + isInvalid={ + getIn(errors, field.name) !== undefined && + getIn(errors, field.name).length > 0 && + getIn(touched, field.name) !== undefined && + getIn(touched, field.name).length > 0 + } + > + + + + + + + + {keyTitle} + + + + + + + + + {valueTitle} + + + + + + + + + + {field.value?.map((mapEntry: InputMapEntry, idx: number) => { + return ( + + + + + <> + + <> + {/** + * We determine if there is an interface based on if there are key options or not, + * as the options would be derived from the underlying interface. + * And if so, these values should be static. + * So, we only display the static text with no mechanism to change it's value. + * Note we still allow more entries, if a user wants to override / add custom + * keys if there is some gaps in the model interface. + */} + {!isEmpty(keyOptions) && + !isEmpty( + getIn( + values, + `${inputMapFieldPath}.${idx}.key` + ) + ) ? ( + + {getIn( + values, + `${inputMapFieldPath}.${idx}.key` + )} + + ) : !isEmpty(keyOptions) ? ( + + ) : ( + + )} + + + + + + + + + + + <> + + <> + {!isEmpty(valueOptions) ? ( + + ) : ( + + )} + + + + { + deleteMapEntry(field.value, idx); + }} + /> + + + + + + + ); + })} + +
+ { + addMapEntry(field.value); + }} + > + {`Add input`} + +
+ + + + ); + }} + + ); } diff --git a/public/utils/config_to_form_utils.ts b/public/utils/config_to_form_utils.ts index c87b7c7d..f5e03d84 100644 --- a/public/utils/config_to_form_utils.ts +++ b/public/utils/config_to_form_utils.ts @@ -16,6 +16,7 @@ import { ConfigFieldValue, ModelFormValue, SearchIndexConfig, + TRANSFORM_TYPE, } from '../../common'; /* @@ -146,7 +147,8 @@ export function getInitialValue(fieldType: ConfigFieldType): ConfigFieldValue { case 'jsonArray': { return '[]'; } - case 'mapArray': { + case 'mapArray': + case 'transformArray': { return []; } case 'boolean': { @@ -155,5 +157,11 @@ export function getInitialValue(fieldType: ConfigFieldType): ConfigFieldValue { case 'number': { return 0; } + case 'transform': { + return { + transformType: TRANSFORM_TYPE.FIELD, + value: '', + }; + } } } diff --git a/public/utils/config_to_schema_utils.ts b/public/utils/config_to_schema_utils.ts index dd5a23d2..7b7a0809 100644 --- a/public/utils/config_to_schema_utils.ts +++ b/public/utils/config_to_schema_utils.ts @@ -192,6 +192,29 @@ export function getFieldSchema( ); break; } + case 'transform': { + yup.object().shape({ + key: defaultStringSchema.required(), + value: yup.object().shape({ + transformType: defaultStringSchema.required(), + value: defaultStringSchema.required(), + }), + }); + + break; + } + case 'transformArray': { + baseSchema = yup.array().of( + yup.object().shape({ + key: defaultStringSchema.required(), + value: yup.object().shape({ + transformType: defaultStringSchema.required(), + value: defaultStringSchema.required(), + }), + }) + ); + break; + } case 'boolean': { baseSchema = yup.boolean(); break; diff --git a/public/utils/config_to_template_utils.ts b/public/utils/config_to_template_utils.ts index 5b9908e4..0a84bf78 100644 --- a/public/utils/config_to_template_utils.ts +++ b/public/utils/config_to_template_utils.ts @@ -25,7 +25,6 @@ import { SearchProcessor, IngestConfig, SearchConfig, - MapFormValue, MapEntry, TEXT_CHUNKING_ALGORITHM, SHARED_OPTIONAL_FIELDS, @@ -33,6 +32,8 @@ import { DELIMITER_OPTIONAL_FIELDS, IngestPipelineConfig, SearchPipelineConfig, + InputMapFormValue, + MapFormValue, } from '../../common'; import { processorConfigToFormik } from './config_to_form_utils'; import { sanitizeJSONPath } from './utils'; @@ -184,7 +185,8 @@ export function processorConfigsToTemplateProcessors( // process input/output maps if (input_map?.length > 0) { processor.ml_inference.input_map = input_map.map( - (mapFormValue: MapFormValue) => mergeMapIntoSingleObj(mapFormValue) + (inputMapFormValue: InputMapFormValue) => + mergeInputMapIntoSingleObj(inputMapFormValue) ); } @@ -442,7 +444,7 @@ export function reduceToTemplate(workflow: Workflow): WorkflowTemplate { // Helper fn to merge the form map (an arr of objs) into a single obj, such that each key // is an obj property, and each value is a property value. Used to format into the -// expected inputs for input_maps and output_maps of the ML inference processors. +// expected inputs for processor configurations function mergeMapIntoSingleObj( mapFormValue: MapFormValue, reverse: boolean = false @@ -462,6 +464,30 @@ function mergeMapIntoSingleObj( return curMap; } +// Same as mergeMapIntoSingleObj, but specified for the ML processor input map format. +function mergeInputMapIntoSingleObj( + mapFormValue: InputMapFormValue, + reverse: boolean = false +): {} { + let curMap = {} as MapEntry; + mapFormValue.forEach((mapEntry) => { + curMap = reverse + ? { + ...curMap, + [sanitizeJSONPath(mapEntry.value.value)]: sanitizeJSONPath( + mapEntry.key + ), + } + : { + ...curMap, + [sanitizeJSONPath(mapEntry.key)]: sanitizeJSONPath( + mapEntry.value.value + ), + }; + }); + return curMap; +} + // utility fn used to build the final set of processor config fields, filtering // by only adding if the field is valid function optionallyAddToFinalForm(