From c6d9adf08fa5d782aa52a8c55b815b8e09842392 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 12 Sep 2024 16:15:30 -0700 Subject: [PATCH] Standardize input/output transforms; bug fixes (#373) Signed-off-by: Tyler Ohlsen --- common/interfaces.ts | 4 +- .../input_fields/boolean_field.tsx | 1 + .../input_transform_modal.tsx | 48 ++++++++++------- .../output_transform_modal.tsx | 54 ++++++++++++------- public/utils/utils.ts | 7 +-- server/routes/helpers.ts | 14 ++++- 6 files changed, 83 insertions(+), 45 deletions(-) diff --git a/common/interfaces.ts b/common/interfaces.ts index 19a4a0bd..9cd0d1b2 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -394,8 +394,8 @@ export type ModelInputFormField = ModelInput & { export type ModelOutputFormField = ModelInputFormField; export type ModelInterface = { - input: ModelInput; - output: ModelOutput; + input?: ModelInput; + output?: ModelOutput; }; export type ConnectorParameters = { diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/boolean_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/boolean_field.tsx index ed391ec7..ba7852f6 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/boolean_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/boolean_field.tsx @@ -59,6 +59,7 @@ export function BooleanField(props: BooleanFieldProps) { } onChange={(id) => { form.setFieldValue(field.name, !field.value); + form.setFieldTouched(field.name, true); }} /> diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx index 2da16980..76901f96 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx @@ -84,6 +84,9 @@ export function InputTransformModal(props: InputTransformModalProps) { const dataSourceId = getDataSourceId(); const { values } = useFormikContext(); + // fetching input data state + const [isFetching, setIsFetching] = useState(false); + // source input / transformed output state const [sourceInput, setSourceInput] = useState('[]'); const [transformedOutput, setTransformedOutput] = useState('{}'); @@ -116,13 +119,7 @@ export function InputTransformModal(props: InputTransformModalProps) { ) { let sampleSourceInput = {}; try { - // In the context of ingest or search resp, this input will be an array (list of docs) - // In the context of request, it will be a single JSON - sampleSourceInput = - props.context === PROCESSOR_CONTEXT.INGEST || - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE - ? JSON.parse(sourceInput)[0] - : JSON.parse(sourceInput); + sampleSourceInput = JSON.parse(sourceInput); const output = generateTransform( sampleSourceInput, map[selectedOutputOption] @@ -167,7 +164,9 @@ export function InputTransformModal(props: InputTransformModalProps) { Source input { + setIsFetching(true); switch (props.context) { case PROCESSOR_CONTEXT.INGEST: { // get the current ingest pipeline up to, but not including, this processor @@ -196,21 +195,31 @@ export function InputTransformModal(props: InputTransformModalProps) { ) .unwrap() .then((resp: SimulateIngestPipelineResponse) => { - setSourceInput(unwrapTransformedDocs(resp)); + const docObjs = unwrapTransformedDocs(resp); + if (docObjs.length > 0) { + setSourceInput(customStringify(docObjs[0])); + } }) .catch((error: any) => { getCore().notifications.toasts.addDanger( `Failed to fetch input data` ); + }) + .finally(() => { + setIsFetching(false); }); } else { try { const docObjs = JSON.parse( values.ingest.docs ) as {}[]; - if (docObjs.length > 0) - setSourceInput(customStringify([docObjs[0]])); - } catch {} + if (docObjs.length > 0) { + setSourceInput(customStringify(docObjs[0])); + } + } catch { + } finally { + setIsFetching(false); + } } break; } @@ -230,6 +239,7 @@ export function InputTransformModal(props: InputTransformModalProps) { if (curSearchPipeline === undefined) { setSourceInput(values.search.request); } + setIsFetching(false); break; } case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { @@ -257,18 +267,20 @@ export function InputTransformModal(props: InputTransformModalProps) { ) .unwrap() .then(async (resp) => { - setSourceInput( - customStringify( - resp.hits.hits.map( - (hit: SearchHit) => hit._source - ) - ) + const hits = resp.hits.hits.map( + (hit: SearchHit) => hit._source ); + if (hits.length > 0) { + setSourceInput(customStringify(hits[0])); + } }) .catch((error: any) => { getCore().notifications.toasts.addDanger( `Failed to fetch source input data` ); + }) + .finally(() => { + setIsFetching(false); }); break; } @@ -357,7 +369,7 @@ export function InputTransformModal(props: InputTransformModalProps) { )} - {outputOptions.length === 1 ? ( + {outputOptions.length <= 1 ? ( Transformed input ) : ( (); + // fetching input data state + const [isFetching, setIsFetching] = useState(false); + // source input / transformed output state const [sourceInput, setSourceInput] = useState('[]'); const [transformedOutput, setTransformedOutput] = useState('{}'); @@ -95,13 +98,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { ) { let sampleSourceInput = {}; try { - // In the context of ingest or search resp, this input will be an array (list of docs) - // In the context of request, it will be a single JSON - sampleSourceInput = - props.context === PROCESSOR_CONTEXT.INGEST || - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE - ? JSON.parse(sourceInput)[0] - : JSON.parse(sourceInput); + sampleSourceInput = JSON.parse(sourceInput); const output = generateTransform( sampleSourceInput, map[selectedOutputOption] @@ -129,7 +126,9 @@ export function OutputTransformModal(props: OutputTransformModalProps) { Source output { + setIsFetching(true); switch (props.context) { // note we skip search request processor context. that is because empty output maps are not supported. // for more details, see comment in ml_processor_inputs.tsx @@ -165,12 +164,24 @@ export function OutputTransformModal(props: OutputTransformModalProps) { ) .unwrap() .then((resp: SimulateIngestPipelineResponse) => { - setSourceInput(unwrapTransformedDocs(resp)); + try { + const docObjs = unwrapTransformedDocs(resp); + if (docObjs.length > 0) { + const sampleModelResult = + docObjs[0]?.inference_results; + setSourceInput( + customStringify(sampleModelResult) + ); + } + } catch {} }) .catch((error: any) => { getCore().notifications.toasts.addDanger( `Failed to fetch input data` ); + }) + .finally(() => { + setIsFetching(false); }); break; } @@ -201,7 +212,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { index: values.ingest.index.name, body: JSON.stringify({ ...JSON.parse(values.search.request as string), - search_pipeline: curSearchPipeline, + search_pipeline: curSearchPipeline || {}, }), }, dataSourceId, @@ -209,18 +220,25 @@ export function OutputTransformModal(props: OutputTransformModalProps) { ) .unwrap() .then(async (resp) => { - setSourceInput( - customStringify( - resp.hits.hits.map( - (hit: SearchHit) => hit._source - ) - ) - ); + const hits = resp.hits.hits.map( + (hit: SearchHit) => hit._source + ) as any[]; + if (hits.length > 0) { + const sampleModelResult = get( + hits, + '0.inference_results.0', + {} + ); + setSourceInput(customStringify(sampleModelResult)); + } }) .catch((error: any) => { getCore().notifications.toasts.addDanger( `Failed to fetch source output data` ); + }) + .finally(() => { + setIsFetching(false); }); break; } @@ -281,7 +299,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) { <> - {outputOptions.length === 1 ? ( + {outputOptions.length <= 1 ? ( Transformed output ) : ( { @@ -167,7 +167,7 @@ export function unwrapTransformedDocs( `Failed to simulate ingest on all documents: ${errorDuringSimulate}` ); } - return customStringify(transformedDocsSources); + return transformedDocsSources; } // ML inference processors will use standard dot notation or JSONPath depending on the input. @@ -181,9 +181,6 @@ export function generateTransform(input: {}, map: MapFormValue): {} { if (mapEntry.value.startsWith(JSONPATH_ROOT_SELECTOR)) { // JSONPath transform transformedResult = jsonpath.query(input, path); - // Non-JSONPath bracket notation not supported - throw an error - } else if (mapEntry.value.includes('[') || mapEntry.value.includes(']')) { - throw new Error(); // Standard dot notation } else { transformedResult = get(input, path); diff --git a/server/routes/helpers.ts b/server/routes/helpers.ts index b66c64dd..87bd4e87 100644 --- a/server/routes/helpers.ts +++ b/server/routes/helpers.ts @@ -12,7 +12,9 @@ import { MODEL_STATE, Model, ModelDict, + ModelInput, ModelInterface, + ModelOutput, NO_MODIFICATIONS_FOUND_TEXT, SearchHit, WORKFLOW_RESOURCE_TYPE, @@ -107,9 +109,17 @@ export function getModelsFromResponses(modelHits: SearchHit[]): ModelDict { | undefined; let modelInterface = undefined as ModelInterface | undefined; if (indexedModelInterface !== undefined) { + let parsedInput = undefined as ModelInput | undefined; + let parsedOutput = undefined as ModelOutput | undefined; + try { + parsedInput = JSON.parse(indexedModelInterface.input); + } catch {} + try { + parsedOutput = JSON.parse(indexedModelInterface.output); + } catch {} modelInterface = { - input: JSON.parse(indexedModelInterface.input), - output: JSON.parse(indexedModelInterface.output), + input: parsedInput, + output: parsedOutput, } as ModelInterface; }