From 18a630a5209a95ffee0e7b88132f5dec292804e8 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 27 Nov 2024 10:08:43 -0800 Subject: [PATCH] Remove legacy modals; clean up ml_processor_inputs Signed-off-by: Tyler Ohlsen --- .../ml_processor_inputs.tsx | 139 +-- .../modals/configure_prompt_modal.tsx | 305 ------- .../ml_processor_inputs/modals/index.ts | 3 - .../modals/input_transform_modal.tsx | 835 ------------------ .../modals/output_transform_modal.tsx | 655 -------------- 5 files changed, 3 insertions(+), 1934 deletions(-) delete mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_prompt_modal.tsx delete mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/input_transform_modal.tsx delete mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/output_transform_modal.tsx diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx index c2dfb6c6..995f39f6 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx @@ -7,7 +7,6 @@ import React, { useState, useEffect } from 'react'; import { getIn, useFormikContext } from 'formik'; import { isEmpty } from 'lodash'; import { useSelector } from 'react-redux'; -import { flattie } from 'flattie'; import { EuiAccordion, EuiCallOut, @@ -25,7 +24,6 @@ import { WorkflowConfig, WorkflowFormValues, ModelInterface, - IndexMappings, EMPTY_INPUT_MAP_ENTRY, REQUEST_PREFIX, REQUEST_PREFIX_WITH_JSONPATH_ROOT_SELECTOR, @@ -39,20 +37,13 @@ import { InputMapFormValue, InputMapArrayFormValue, } from '../../../../../../common'; -import { - ConfigurePromptModal, - InputTransformModal, - OutputTransformModal, - OverrideQueryModal, -} from './modals'; +import { OverrideQueryModal } from './modals'; import { ModelInputs } from './model_inputs'; -import { AppState, getMappings, useAppDispatch } from '../../../../../store'; +import { AppState } from '../../../../../store'; import { formikToPartialPipeline, - getDataSourceId, parseModelInputs, parseModelOutputs, - sanitizeJSONPath, } from '../../../../../utils'; import { ConfigFieldList } from '../../config_field_list'; import { ModelOutputs } from './model_outputs'; @@ -71,10 +62,7 @@ interface MLProcessorInputsProps { * output map configuration forms, respectively. */ export function MLProcessorInputs(props: MLProcessorInputsProps) { - const dispatch = useAppDispatch(); - const dataSourceId = getDataSourceId(); - const { models, connectors } = useSelector((state: AppState) => state.ml); - const indices = useSelector((state: AppState) => state.opensearch.indices); + const { models } = useSelector((state: AppState) => state.ml); const { values, setFieldValue, setFieldTouched } = useFormikContext< WorkflowFormValues >(); @@ -115,13 +103,6 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { }, [props.uiConfig.search.enrichRequest.processors]); // various modal states - const [isInputTransformModalOpen, setIsInputTransformModalOpen] = useState< - boolean - >(false); - const [isOutputTransformModalOpen, setIsOutputTransformModalOpen] = useState< - boolean - >(false); - const [isPromptModalOpen, setIsPromptModalOpen] = useState(false); const [isQueryModalOpen, setIsQueryModalOpen] = useState(false); // model interface state @@ -168,122 +149,8 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { } }, [models]); - // persisting doc/query/index mapping fields to collect a list - // of options to display in the dropdowns when configuring input / output maps - const [docFields, setDocFields] = useState<{ label: string }[]>([]); - const [queryFields, setQueryFields] = useState<{ label: string }[]>([]); - const [indexMappingFields, setIndexMappingFields] = useState< - { label: string }[] - >([]); - useEffect(() => { - try { - const docObjKeys = Object.keys( - flattie((JSON.parse(values.ingest.docs) as {}[])[0]) - ); - if (docObjKeys.length > 0) { - setDocFields( - docObjKeys.map((key) => { - return { - label: - // ingest inputs can handle dot notation, and hence don't need - // sanitizing to handle JSONPath edge cases. The other contexts - // only support JSONPath, and hence need some post-processing/sanitizing. - props.context === PROCESSOR_CONTEXT.INGEST - ? key - : sanitizeJSONPath(key), - }; - }) - ); - } - } catch {} - }, [values?.ingest?.docs]); - useEffect(() => { - try { - const queryObjKeys = Object.keys( - flattie(JSON.parse(values.search.request)) - ); - if (queryObjKeys.length > 0) { - setQueryFields( - queryObjKeys.map((key) => { - return { - label: - // ingest inputs can handle dot notation, and hence don't need - // sanitizing to handle JSONPath edge cases. The other contexts - // only support JSONPath, and hence need some post-processing/sanitizing. - props.context === PROCESSOR_CONTEXT.INGEST - ? key - : sanitizeJSONPath(key), - }; - }) - ); - } - } catch {} - }, [values?.search?.request]); - useEffect(() => { - const indexName = values?.search?.index?.name as string | undefined; - if (indexName !== undefined && indices[indexName] !== undefined) { - dispatch( - getMappings({ - index: indexName, - dataSourceId, - }) - ) - .unwrap() - .then((resp: IndexMappings) => { - const mappingsObjKeys = Object.keys(resp.properties); - if (mappingsObjKeys.length > 0) { - setIndexMappingFields( - mappingsObjKeys.map((key) => { - return { - label: key, - type: resp.properties[key]?.type, - }; - }) - ); - } - }); - } - }, [values?.search?.index?.name]); - return ( <> - {isInputTransformModalOpen && ( - setIsInputTransformModalOpen(false)} - /> - )} - {isOutputTransformModalOpen && ( - setIsOutputTransformModalOpen(false)} - /> - )} - {isPromptModalOpen && ( - setIsPromptModalOpen(false)} - /> - )} {isQueryModalOpen && ( void; -} - -/** - * A modal to configure a prompt template. Can manually configure, include placeholder values - * using other model inputs, and/or select from a presets library. - */ -export function ConfigurePromptModal(props: ConfigurePromptModalProps) { - const { values, setFieldValue, setFieldTouched } = useFormikContext< - WorkflowFormValues - >(); - - // get some current form values - const modelConfigPath = `${props.baseConfigPath}.${props.config.id}.model_config`; - const modelConfig = getIn(values, modelConfigPath) as string; - const modelInputs = parseModelInputs(props.modelInterface); - - // popover states - const [schemaPopoverOpen, setSchemaPopoverOpen] = useState(false); - const [presetsPopoverOpen, setPresetsPopoverOpen] = useState(false); - - // prompt str state. manipulated as users manually update, or - // from selecting a preset - const [promptStr, setPromptStr] = useState(''); - - // hook to set the prompt if found in the model config - useEffect(() => { - try { - const modelConfigObj = JSON.parse(getIn(values, modelConfigPath)); - const prompt = getIn(modelConfigObj, PROMPT_FIELD); - if (!isEmpty(prompt)) { - setPromptStr(prompt); - } else { - setPromptStr(''); - } - } catch {} - }, [getIn(values, modelConfigPath)]); - - return ( - - - -

{`Configure prompt`}

- - - - - Configure a custom prompt template for the model. Optionally inject - dynamic model inputs into the template. - - - - <> - - setPresetsPopoverOpen(!presetsPopoverOpen)} - iconSide="right" - iconType="arrowDown" - > - Choose from a preset - - } - isOpen={presetsPopoverOpen} - closePopover={() => setPresetsPopoverOpen(false)} - anchorPosition="downLeft" - > - ({ - name: preset.name, - onClick: () => { - try { - setFieldValue( - modelConfigPath, - customStringify({ - ...JSON.parse(modelConfig), - [PROMPT_FIELD]: preset.prompt, - }) - ); - } catch {} - setFieldTouched(modelConfigPath, true); - setPresetsPopoverOpen(false); - }, - })), - }, - ]} - /> - - - Prompt - - setPromptStr(value)} - onBlur={(e) => { - let updatedModelConfig = {} as any; - try { - updatedModelConfig = JSON.parse(modelConfig); - } catch {} - if (isEmpty(promptStr)) { - // if the input is blank, it is assumed the user - // does not want any prompt. hence, remove the "prompt" field - // from the config altogether. - delete updatedModelConfig[PROMPT_FIELD]; - } else { - updatedModelConfig[PROMPT_FIELD] = promptStr; - } - setFieldValue( - modelConfigPath, - customStringify(updatedModelConfig) - ); - setFieldTouched(modelConfigPath); - }} - /> - {modelInputs.length > 0 && ( - <> - - - <> - - - To use any model inputs in the prompt template, copy the - placeholder string directly. - - - - - setSchemaPopoverOpen(false)} - panelPaddingSize="s" - button={ - - setSchemaPopoverOpen(!schemaPopoverOpen) - } - > - View input schema - - } - > - - The JSON Schema defining the model's expected input - - - {customStringify( - parseModelInputsObj(props.modelInterface) - )} - - - - - - - )} - - - - - - - Close - - - - ); -} - -const columns = [ - { - name: 'Name', - field: 'label', - width: '25%', - }, - { - name: 'Type', - field: 'type', - width: '15%', - }, - { - name: 'Placeholder string', - field: 'label', - width: '50%', - render: (label: string, modelInput: ModelInputFormField) => ( - - {getPlaceholderString(modelInput.type, label)} - - ), - }, - { - name: 'Actions', - field: 'label', - width: '10%', - render: (label: string, modelInput: ModelInputFormField) => ( - - {(copy) => ( - - )} - - ), - }, -]; - -// small util fn to get the full placeholder string to be -// inserted into the template. String conversion is required -// if the input is an array, for example. Also, all values -// should be prepended with "parameters.", as all inputs -// will be nested under a base parameters obj. -function getPlaceholderString(type: string, label: string) { - return type === 'array' - ? `\$\{parameters.${label}.toString()\}` - : `\$\{parameters.${label}\}`; -} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/index.ts b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/index.ts index 8d0de609..c6c153c2 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/index.ts +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/index.ts @@ -3,9 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './input_transform_modal'; -export * from './output_transform_modal'; -export * from './configure_prompt_modal'; export * from './override_query_modal'; export * from './configure_template_modal'; export * from './configure_expression_modal'; diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/input_transform_modal.tsx deleted file mode 100644 index e3ddf326..00000000 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/input_transform_modal.tsx +++ /dev/null @@ -1,835 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useState, useEffect } from 'react'; -import { useFormikContext, getIn, Formik } from 'formik'; -import { isEmpty } from 'lodash'; -import Ajv from 'ajv'; -import * as yup from 'yup'; -import { - EuiCodeEditor, - EuiFlexGroup, - EuiFlexItem, - EuiModal, - EuiModalBody, - EuiModalFooter, - EuiModalHeader, - EuiModalHeaderTitle, - EuiCompressedSelect, - EuiSelectOption, - EuiSmallButton, - EuiSpacer, - EuiText, - EuiPopover, - EuiSmallButtonEmpty, - EuiCodeBlock, - EuiPopoverTitle, - EuiIconTip, - EuiCompressedSwitch, - EuiCallOut, - EuiAccordion, -} from '@elastic/eui'; -import { - IConfigField, - IProcessorConfig, - IngestPipelineConfig, - InputTransformFormValues, - MapArrayFormValue, - ModelInterface, - PROCESSOR_CONTEXT, - SearchHit, - SimulateIngestPipelineResponse, - TRANSFORM_CONTEXT, - WorkflowConfig, - WorkflowFormValues, - customStringify, - REQUEST_PREFIX, - REQUEST_PREFIX_WITH_JSONPATH_ROOT_SELECTOR, -} from '../../../../../../../common'; -import { - formikToPartialPipeline, - generateTransform, - getFieldSchema, - getInitialValue, - prepareDocsForSimulate, - unwrapTransformedDocs, -} from '../../../../../../utils'; -import { - searchIndex, - simulatePipeline, - useAppDispatch, -} from '../../../../../../store'; -import { getCore } from '../../../../../../services'; -import { - generateArrayTransform, - getDataSourceId, - parseModelInputs, - parseModelInputsObj, -} from '../../../../../../utils/utils'; -import { BooleanField, MapArrayField } from '../../../input_fields'; - -interface InputTransformModalProps { - uiConfig: WorkflowConfig; - config: IProcessorConfig; - baseConfigPath: string; - context: PROCESSOR_CONTEXT; - inputMapFieldPath: string; - modelInterface: ModelInterface | undefined; - valueOptions: { label: string }[]; - onClose: () => void; -} - -// the max number of input docs we use to display & test transforms with (search response hits) -const MAX_INPUT_DOCS = 10; - -/** - * A modal to configure advanced JSON-to-JSON transforms into a model's expected input - */ -export function InputTransformModal(props: InputTransformModalProps) { - const dispatch = useAppDispatch(); - const dataSourceId = getDataSourceId(); - const { values, setFieldValue, setFieldTouched } = useFormikContext< - WorkflowFormValues - >(); - - // sub-form values/schema - const inputTransformFormValues = { - input_map: getInitialValue('mapArray'), - one_to_one: getInitialValue('boolean'), - } as InputTransformFormValues; - const inputTransformFormSchema = yup.object({ - input_map: getFieldSchema({ - type: 'mapArray', - } as IConfigField), - one_to_one: getFieldSchema( - { - type: 'boolean', - } as IConfigField, - true - ), - }) as yup.Schema; - - // persist standalone values. update / initialize when it is first opened - const [tempErrors, setTempErrors] = useState(false); - const [tempOneToOne, setTempOneToOne] = useState(false); - const [tempInputMap, setTempInputMap] = useState([]); - - // various prompt states - const [viewPromptDetails, setViewPromptDetails] = useState(false); - const [viewTransformedPrompt, setViewTransformedPrompt] = useState( - false - ); - const [originalPrompt, setOriginalPrompt] = useState(''); - const [transformedPrompt, setTransformedPrompt] = useState(''); - - // fetching input data state - const [isFetching, setIsFetching] = useState(false); - - // source input / transformed input state - const [sourceInput, setSourceInput] = useState('{}'); - const [transformedInput, setTransformedInput] = useState('{}'); - - // get some current form values - const oneToOnePath = `${props.baseConfigPath}.${props.config.id}.one_to_one`; - const docs = getIn(values, 'ingest.docs'); - let docObjs = [] as {}[] | undefined; - try { - docObjs = JSON.parse(docs); - } catch {} - const query = getIn(values, 'search.request'); - let queryObj = {} as {} | undefined; - try { - queryObj = JSON.parse(query); - } catch {} - const onIngestAndNoDocs = - props.context === PROCESSOR_CONTEXT.INGEST && isEmpty(docObjs); - const onSearchAndNoQuery = - (props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST || - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && - isEmpty(queryObj); - - // selected transform state - const transformOptions = tempInputMap.map((_, idx) => ({ - value: idx, - text: `Prediction ${idx + 1}`, - })) as EuiSelectOption[]; - const [selectedTransformOption, setSelectedTransformOption] = useState< - number - >((transformOptions[0]?.value as number) ?? 0); - - // popover state containing the model interface details, if applicable - const [popoverOpen, setPopoverOpen] = useState(false); - - // validation state utilizing the model interface, if applicable. undefined if - // there is no model interface and/or no source input - const [isValid, setIsValid] = useState(undefined); - - const description = - props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST - ? 'Fetch an input query and see how it is transformed.' - : props.context === PROCESSOR_CONTEXT.INGEST - ? 'Fetch a sample document and see how it is transformed' - : `Fetch some returned documents (up to ${MAX_INPUT_DOCS}) and see how they are transformed.`; - - // hook to re-generate the transform when any inputs to the transform are updated - useEffect(() => { - if (!isEmpty(tempInputMap) && !isEmpty(JSON.parse(sourceInput))) { - let sampleSourceInput = {} as {} | []; - try { - sampleSourceInput = JSON.parse(sourceInput); - const output = - // Edge case: users are collapsing input docs into a single input field when many-to-one is selected - // fo input transforms on search response processors. - tempOneToOne === false && - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE && - Array.isArray(sampleSourceInput) - ? generateArrayTransform( - sampleSourceInput as [], - tempInputMap[selectedTransformOption], - props.context, - TRANSFORM_CONTEXT.INPUT, - queryObj - ) - : generateTransform( - sampleSourceInput, - tempInputMap[selectedTransformOption], - props.context, - TRANSFORM_CONTEXT.INPUT, - queryObj - ); - - setTransformedInput(customStringify(output)); - } catch {} - } else { - setTransformedInput('{}'); - } - }, [tempInputMap, sourceInput, selectedTransformOption]); - - // hook to re-determine validity when the generated output changes - // utilize Ajv JSON schema validator library. For more info/examples, see - // https://www.npmjs.com/package/ajv - useEffect(() => { - if ( - !isEmpty(JSON.parse(sourceInput)) && - !isEmpty(props.modelInterface?.input?.properties?.parameters) - ) { - const validateFn = new Ajv().compile( - props.modelInterface?.input?.properties?.parameters || {} - ); - setIsValid(validateFn(JSON.parse(transformedInput))); - } else { - setIsValid(undefined); - } - }, [transformedInput]); - - // hook to set the prompt if found in the model config - useEffect(() => { - const modelConfigString = getIn( - values, - `${props.baseConfigPath}.${props.config.id}.model_config` - ); - try { - const prompt = JSON.parse(modelConfigString)?.prompt; - if (!isEmpty(prompt)) { - setOriginalPrompt(prompt); - } - } catch {} - }, [ - getIn(values, `${props.baseConfigPath}.${props.config.id}.model_config`), - ]); - - // hook to set the transformed prompt, if a valid prompt found, and - // valid parameters set - useEffect(() => { - const transformedInputObj = JSON.parse(transformedInput); - if (!isEmpty(originalPrompt) && !isEmpty(transformedInputObj)) { - setTransformedPrompt( - injectValuesIntoPrompt(originalPrompt, transformedInputObj) - ); - setViewPromptDetails(true); - setViewTransformedPrompt(true); - } else { - setViewPromptDetails(false); - setViewTransformedPrompt(false); - setTransformedPrompt(originalPrompt); - } - }, [originalPrompt, transformedInput]); - - // hook to clear the source input when one_to_one is toggled - useEffect(() => { - setSourceInput('{}'); - }, [tempOneToOne]); - - return ( - {}} - validate={(values) => {}} - > - {(formikProps) => { - // override to parent form values when changes detected - useEffect(() => { - formikProps.setFieldValue( - 'input_map', - getIn(values, props.inputMapFieldPath) - ); - }, [getIn(values, props.inputMapFieldPath)]); - useEffect(() => { - formikProps.setFieldValue('one_to_one', getIn(values, oneToOnePath)); - }, [getIn(values, oneToOnePath)]); - - // update temp vars when form changes are detected - useEffect(() => { - setTempInputMap(getIn(formikProps.values, 'input_map')); - }, [getIn(formikProps.values, 'input_map')]); - useEffect(() => { - setTempOneToOne(getIn(formikProps.values, 'one_to_one')); - }, [getIn(formikProps.values, 'one_to_one')]); - - // update tempErrors if errors detected - useEffect(() => { - setTempErrors(!isEmpty(formikProps.errors)); - }, [formikProps.errors]); - - const InputMap = ( - { - if (isEmpty(curArray)) { - setSelectedTransformOption(0); - } - }} - // If the map we are deleting is the one we last used to test, reset the state and - // default to the first map in the list. - onMapDelete={(idxToDelete) => { - if (selectedTransformOption === idxToDelete) { - setSelectedTransformOption(0); - setTransformedInput('{}'); - } - }} - addMapEntryButtonText="Add input" - addMapButtonText="Add input group (Advanced)" - mappingDirection="sortLeft" - /> - ); - - const OneToOneConfig = ( - - ); - - const FetchButton = ( - { - setIsFetching(true); - switch (props.context) { - case PROCESSOR_CONTEXT.INGEST: { - // get the current ingest pipeline up to, but not including, this processor - const curIngestPipeline = formikToPartialPipeline( - values, - props.uiConfig, - props.config.id, - false, - PROCESSOR_CONTEXT.INGEST - ); - // if there are preceding processors, we need to simulate the partial ingest pipeline, - // in order to get the latest transformed version of the docs - if (curIngestPipeline !== undefined) { - const curDocs = prepareDocsForSimulate( - values.ingest.docs, - values.ingest.index.name - ); - await dispatch( - simulatePipeline({ - apiBody: { - pipeline: curIngestPipeline as IngestPipelineConfig, - docs: [curDocs[0]], - }, - dataSourceId, - }) - ) - .unwrap() - .then((resp: SimulateIngestPipelineResponse) => { - const docObjs = unwrapTransformedDocs(resp); - if (docObjs.length > 0) { - setSourceInput(customStringify(docObjs[0])); - } - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch input data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - } else { - try { - const docObjs = JSON.parse(values.ingest.docs) as {}[]; - if (docObjs.length > 0) { - setSourceInput(customStringify(docObjs[0])); - } - } catch { - } finally { - setIsFetching(false); - } - } - break; - } - case PROCESSOR_CONTEXT.SEARCH_REQUEST: { - // get the current search pipeline up to, but not including, this processor - const curSearchPipeline = formikToPartialPipeline( - values, - props.uiConfig, - props.config.id, - false, - PROCESSOR_CONTEXT.SEARCH_REQUEST - ); - // if there are preceding processors, we cannot generate. The button to render - // this modal should be disabled if the search pipeline would be enabled. We add - // this if check as an extra layer of checking, and if mechanism for gating - // this is changed in the future. - if (curSearchPipeline === undefined) { - setSourceInput(values.search.request); - } - setIsFetching(false); - break; - } - case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { - // get the current search pipeline up to, but not including, this processor - const curSearchPipeline = formikToPartialPipeline( - values, - props.uiConfig, - props.config.id, - false, - PROCESSOR_CONTEXT.SEARCH_RESPONSE - ); - // Execute search. If there are preceding processors, augment the existing query with - // the partial search pipeline (inline) to get the latest transformed version of the response. - dispatch( - searchIndex({ - apiBody: { - index: values.search.index.name, - body: JSON.stringify({ - ...JSON.parse(values.search.request as string), - search_pipeline: curSearchPipeline || {}, - }), - }, - dataSourceId, - }) - ) - .unwrap() - .then(async (resp) => { - const hits = resp?.hits?.hits - ?.map((hit: SearchHit) => hit._source) - .slice(0, MAX_INPUT_DOCS); - if (hits.length > 0) { - setSourceInput( - // if one-to-one, treat the source input as a single retrieved document - // else, treat it as all of the returned documents - customStringify(tempOneToOne ? hits[0] : hits) - ); - } - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch source input data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - break; - } - } - }} - > - Fetch data - - ); - - const SourceInput = ( - - ); - - const TransformedInput = ( - - ); - - return ( - - - -

{`Preview input transformation`}

- - - - - - <> - - - -

Define transform

- - - - - - - - {InputMap} - {props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE && ( - <> - - - - - {OneToOneConfig} - - - - )} - - - - - - -

Preview

- - - - - - - - <> - {(onIngestAndNoDocs || onSearchAndNoQuery) && ( - <> - - - - )} - {FetchButton} - - - - - - <> - - - - Data before transformation - - - - - {SourceInput} - - - - <> - - {isValid !== undefined && ( - - - - )} - - {transformOptions.length <= 1 ? ( - - Data after transformation - - ) : ( - - Data after transformation for - - } - options={transformOptions} - value={selectedTransformOption} - onChange={(e) => { - setSelectedTransformOption( - Number(e.target.value) - ); - }} - /> - )} - - {!isEmpty( - parseModelInputsObj(props.modelInterface) - ) && ( - - setPopoverOpen(false)} - panelPaddingSize="s" - button={ - setPopoverOpen(!popoverOpen)} - > - Input schema - - } - > - - The JSON Schema defining the model's expected - input - - - {customStringify( - parseModelInputsObj(props.modelInterface) - )} - - - - )} - - - {TransformedInput} - - - - - {!isEmpty(originalPrompt) && ( - - <> - - - Transformed prompt - - - - setViewPromptDetails(!viewPromptDetails) - } - disabled={isEmpty(JSON.parse(transformedInput))} - /> - - {isEmpty(JSON.parse(transformedInput)) && ( - - - Transformed input is empty - - - )} - - {viewPromptDetails && ( - <> - - - setViewTransformedPrompt(!viewTransformedPrompt) - } - /> - - - - )} - - - )} - - - - - Cancel - - { - // update the parent form values - if (props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) { - setFieldValue( - oneToOnePath, - getIn(formikProps.values, 'one_to_one') - ); - setFieldTouched(oneToOnePath, true); - } - setFieldValue( - props.inputMapFieldPath, - getIn(formikProps.values, 'input_map') - ); - setFieldTouched(props.inputMapFieldPath, true); - props.onClose(); - }} - isDisabled={tempErrors} // blocking update until valid input is given - fill={true} - color="primary" - data-testid="updateInputTransformModalButton" - > - Save - - - - ); - }} - - ); -} - -function injectValuesIntoPrompt( - promptString: string, - parameters: { [key: string]: string } -): string { - let finalPromptString = promptString; - // replace any parameter placeholders in the prompt with any values found in the - // parameters obj. - // we do 2 checks - one for the regular prompt, and one with "toString()" appended. - // this is required for parameters that have values as a list, for example. - Object.keys(parameters).forEach((parameterKey) => { - const parameterValue = parameters[parameterKey]; - const regex = new RegExp(`\\$\\{parameters.${parameterKey}\\}`, 'g'); - const regexWithToString = new RegExp( - `\\$\\{parameters.${parameterKey}.toString\\(\\)\\}`, - 'g' - ); - finalPromptString = finalPromptString - .replace(regex, parameterValue) - .replace(regexWithToString, parameterValue); - }); - - return finalPromptString; -} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/output_transform_modal.tsx deleted file mode 100644 index 2e43e4fe..00000000 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/output_transform_modal.tsx +++ /dev/null @@ -1,655 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useState, useEffect } from 'react'; -import { useFormikContext, getIn, Formik } from 'formik'; -import { cloneDeep, isEmpty, set } from 'lodash'; -import * as yup from 'yup'; -import { - EuiCodeEditor, - EuiFlexGroup, - EuiFlexItem, - EuiModal, - EuiModalBody, - EuiModalFooter, - EuiModalHeader, - EuiModalHeaderTitle, - EuiCompressedSelect, - EuiSelectOption, - EuiSmallButton, - EuiSpacer, - EuiText, - EuiPopover, - EuiSmallButtonEmpty, - EuiPopoverTitle, - EuiCodeBlock, - EuiCallOut, - EuiIconTip, - EuiAccordion, -} from '@elastic/eui'; -import { - IConfigField, - IProcessorConfig, - IngestPipelineConfig, - MapArrayFormValue, - MapFormValue, - ModelInterface, - OutputTransformFormValues, - PROCESSOR_CONTEXT, - SearchHit, - SearchPipelineConfig, - SimulateIngestPipelineResponse, - TRANSFORM_CONTEXT, - WorkflowConfig, - WorkflowFormValues, - customStringify, -} from '../../../../../../../common'; -import { - formikToPartialPipeline, - generateTransform, - getFieldSchema, - getInitialValue, - prepareDocsForSimulate, - unwrapTransformedDocs, -} from '../../../../../../utils'; -import { - searchIndex, - simulatePipeline, - useAppDispatch, -} from '../../../../../../store'; -import { getCore } from '../../../../../../services'; -import { BooleanField, MapArrayField } from '../../../input_fields'; -import { - getDataSourceId, - parseModelOutputs, - parseModelOutputsObj, -} from '../../../../../../utils/utils'; - -interface OutputTransformModalProps { - uiConfig: WorkflowConfig; - config: IProcessorConfig; - baseConfigPath: string; - context: PROCESSOR_CONTEXT; - outputMapFieldPath: string; - modelInterface: ModelInterface | undefined; - onClose: () => void; -} - -/** - * A modal to configure advanced JSON-to-JSON transforms from a model's expected output - */ -export function OutputTransformModal(props: OutputTransformModalProps) { - const dispatch = useAppDispatch(); - const dataSourceId = getDataSourceId(); - const { values, setFieldValue, setFieldTouched } = useFormikContext< - WorkflowFormValues - >(); - - // sub-form values/schema - const outputTransformFormValues = { - output_map: getInitialValue('mapArray'), - full_response_path: getInitialValue('boolean'), - } as OutputTransformFormValues; - const outputTransformFormSchema = yup.object({ - output_map: getFieldSchema({ - type: 'mapArray', - } as IConfigField), - full_response_path: getFieldSchema( - { - type: 'boolean', - } as IConfigField, - true - ), - }) as yup.Schema; - - // persist standalone values. update / initialize when it is first opened - const [tempErrors, setTempErrors] = useState(false); - const [tempFullResponsePath, setTempFullResponsePath] = useState( - false - ); - const [tempOutputMap, setTempOutputMap] = useState([]); - - // fetching input data state - const [isFetching, setIsFetching] = useState(false); - - // source output / transformed output state - const [sourceOutput, setSourceOutput] = useState('{}'); - const [transformedOutput, setTransformedOutput] = useState('{}'); - - // get some current form values - const fullResponsePathPath = `${props.baseConfigPath}.${props.config.id}.full_response_path`; - const docs = getIn(values, 'ingest.docs'); - let docObjs = [] as {}[] | undefined; - try { - docObjs = JSON.parse(docs); - } catch {} - const query = getIn(values, 'search.request'); - let queryObj = {} as {} | undefined; - try { - queryObj = JSON.parse(query); - } catch {} - const onIngestAndNoDocs = - props.context === PROCESSOR_CONTEXT.INGEST && isEmpty(docObjs); - const onSearchAndNoQuery = - (props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST || - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && - isEmpty(queryObj); - - // popover state containing the model interface details, if applicable - const [popoverOpen, setPopoverOpen] = useState(false); - - // selected transform state - const transformOptions = tempOutputMap.map((_, idx) => ({ - value: idx, - text: `Prediction ${idx + 1}`, - })) as EuiSelectOption[]; - const [selectedTransformOption, setSelectedTransformOption] = useState< - number - >((transformOptions[0]?.value as number) ?? 0); - - // hook to re-generate the transform when any inputs to the transform are updated - useEffect(() => { - if (!isEmpty(tempOutputMap) && !isEmpty(JSON.parse(sourceOutput))) { - let sampleSourceOutput = {}; - try { - sampleSourceOutput = JSON.parse(sourceOutput); - const output = generateTransform( - sampleSourceOutput, - reverseKeysAndValues(tempOutputMap[selectedTransformOption]), - props.context, - TRANSFORM_CONTEXT.OUTPUT - ); - setTransformedOutput(customStringify(output)); - } catch {} - } else { - setTransformedOutput('{}'); - } - }, [tempOutputMap, sourceOutput, selectedTransformOption]); - - // hook to clear the source output when full_response_path is toggled - useEffect(() => { - setSourceOutput('{}'); - }, [tempFullResponsePath]); - - return ( - {}} - validate={(values) => {}} - > - {(formikProps) => { - // override to parent form values when changes detected - useEffect(() => { - formikProps.setFieldValue( - 'output_map', - getIn(values, props.outputMapFieldPath) - ); - }, [getIn(values, props.outputMapFieldPath)]); - useEffect(() => { - formikProps.setFieldValue( - 'full_response_path', - getIn(values, fullResponsePathPath) - ); - }, [getIn(values, fullResponsePathPath)]); - - // update temp vars when form changes are detected - useEffect(() => { - setTempOutputMap(getIn(formikProps.values, 'output_map')); - }, [getIn(formikProps.values, 'output_map')]); - useEffect(() => { - setTempFullResponsePath( - getIn(formikProps.values, 'full_response_path') - ); - }, [getIn(formikProps.values, 'full_response_path')]); - - // update tempErrors if errors detected - useEffect(() => { - setTempErrors(!isEmpty(formikProps.errors)); - }, [formikProps.errors]); - - const OutputMap = ( - { - if (isEmpty(curArray)) { - setSelectedTransformOption(0); - } - }} - // If the map we are deleting is the one we last used to test, reset the state and - // default to the first map in the list. - onMapDelete={(idxToDelete) => { - if (selectedTransformOption === idxToDelete) { - setSelectedTransformOption(0); - setTransformedOutput('{}'); - } - }} - addMapEntryButtonText="Add output" - addMapButtonText="Add output group (Advanced)" - mappingDirection="sortRight" - /> - ); - - const FullResponsePathConfig = ( - - ); - - const FetchButton = ( - { - setIsFetching(true); - switch (props.context) { - // note we skip search request processor context. that is because empty output maps are not supported. - // for more details, see comment in ml_processor_inputs.tsx - case PROCESSOR_CONTEXT.INGEST: { - // get the current ingest pipeline up to, and including this processor. - // remove any currently-configured output map since we only want the transformation - // up to, and including, the input map transformations - const valuesWithoutOutputMapConfig = cloneDeep(values); - set( - valuesWithoutOutputMapConfig, - props.outputMapFieldPath, - [] - ); - set( - valuesWithoutOutputMapConfig, - fullResponsePathPath, - getIn(formikProps.values, 'full_response_path') - ); - const curIngestPipeline = formikToPartialPipeline( - valuesWithoutOutputMapConfig, - props.uiConfig, - props.config.id, - true, - PROCESSOR_CONTEXT.INGEST - ) as IngestPipelineConfig; - const curDocs = prepareDocsForSimulate( - values.ingest.docs, - values.ingest.index.name - ); - await dispatch( - simulatePipeline({ - apiBody: { - pipeline: curIngestPipeline, - docs: [curDocs[0]], - }, - dataSourceId, - }) - ) - .unwrap() - .then((resp: SimulateIngestPipelineResponse) => { - try { - const docObjs = unwrapTransformedDocs(resp); - if (docObjs.length > 0) { - const sampleModelResult = - docObjs[0]?.inference_results || {}; - setSourceOutput(customStringify(sampleModelResult)); - } - } catch {} - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch input data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - break; - } - case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { - // get the current search pipeline up to, and including this processor. - // remove any currently-configured output map since we only want the transformation - // up to, and including, the input map transformations - const valuesWithoutOutputMapConfig = cloneDeep(values); - set( - valuesWithoutOutputMapConfig, - props.outputMapFieldPath, - [] - ); - set( - valuesWithoutOutputMapConfig, - fullResponsePathPath, - getIn(formikProps.values, 'full_response_path') - ); - const curSearchPipeline = formikToPartialPipeline( - valuesWithoutOutputMapConfig, - props.uiConfig, - props.config.id, - true, - PROCESSOR_CONTEXT.SEARCH_RESPONSE - ) as SearchPipelineConfig; - - // Execute search. Augment the existing query with - // the partial search pipeline (inline) to get the latest transformed - // version of the request. - dispatch( - searchIndex({ - apiBody: { - index: values.ingest.index.name, - body: JSON.stringify({ - ...JSON.parse(values.search.request as string), - search_pipeline: curSearchPipeline || {}, - }), - }, - dataSourceId, - }) - ) - .unwrap() - .then(async (resp) => { - const hits = resp?.hits?.hits?.map( - (hit: SearchHit) => hit._source - ) as any[]; - if (hits.length > 0) { - const sampleModelResult = - hits[0].inference_results || {}; - setSourceOutput(customStringify(sampleModelResult)); - } - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch source output data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - break; - } - } - }} - > - Fetch data - - ); - - const SourceOutput = ( - - ); - - const TransformedOutput = ( - - ); - - return ( - - - -

{`Preview output transformation`}

- - - - - - <> - - - -

Define transform

- - - - - - - - {OutputMap} - {(props.context === PROCESSOR_CONTEXT.INGEST || - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && ( - <> - - - - - {FullResponsePathConfig} - - - - )} - - - - - - -

Preview

- - - - - - - - <> - {(onIngestAndNoDocs || onSearchAndNoQuery) && ( - <> - - - - )} - {FetchButton} - - - - - - <> - - - - - Data before transformation - - - {!isEmpty( - parseModelOutputsObj( - props.modelInterface, - tempFullResponsePath - ) - ) && ( - - setPopoverOpen(false)} - panelPaddingSize="s" - button={ - - setPopoverOpen(!popoverOpen) - } - > - Output schema - - } - > - - The JSON Schema defining the model's - expected output - - - {customStringify( - parseModelOutputsObj( - props.modelInterface, - tempFullResponsePath - ) - )} - - - - )} - - - {SourceOutput} - - - - - <> - {transformOptions.length <= 1 ? ( - Data after transformation - ) : ( - - Data after transformation for - - } - options={transformOptions} - value={selectedTransformOption} - onChange={(e) => { - setSelectedTransformOption( - Number(e.target.value) - ); - }} - /> - )} - - - - {TransformedOutput} - - - - - - - - - Cancel - - { - // update the parent form values - setFieldValue( - fullResponsePathPath, - getIn(formikProps.values, 'full_response_path') - ); - setFieldTouched(fullResponsePathPath, true); - - setFieldValue( - props.outputMapFieldPath, - getIn(formikProps.values, 'output_map') - ); - setFieldTouched(props.outputMapFieldPath, true); - props.onClose(); - }} - isDisabled={tempErrors} // blocking update until valid input is given - fill={true} - color="primary" - data-testid="updateOutputTransformModalButton" - > - Save - - - - ); - }} - - ); -} - -// since we persist the form keys/values reversed, we have a util fn to reverse back, so we can use -// single util fns for manipulation of the form values (generating transforms, etc.) -function reverseKeysAndValues(values: MapFormValue): MapFormValue { - return values.map((mapEntry) => { - return { - key: mapEntry.value, - value: mapEntry.key, - }; - }); -}