Skip to content

Commit

Permalink
Integrate with JSONPath; complete input transform (ingest) (opensearc…
Browse files Browse the repository at this point in the history 
…h-project#229)

Signed-off-by: Tyler Ohlsen <[email protected]>
  • Loading branch information
@ohltyler
ohltyler authored Jul 22, 2024
1 parent 6dc64bb commit d490d15
Show file tree
Hide file tree
Showing 9 changed files with 337 additions and 100 deletions.
1 change: 1 addition & 0 deletions common/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ export const FETCH_ALL_QUERY_BODY = {
size: 1000,
};
export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception';
export const JSONPATH_ROOT_SELECTOR = '$.';

export enum PROCESSOR_CONTEXT {
INGEST = 'ingest',
Expand Down
4 changes: 3 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,13 @@
]
},
"dependencies": {
"@types/jsonpath": "^0.2.4",
"formik": "2.4.2",
"js-yaml": "^4.1.0",
"jsonpath": "^1.1.1",
"reactflow": "^11.8.3",
"yup": "^1.3.2"
},
"devDependencies": {},
"resolutions": {}
}
}
25 changes: 0 additions & 25 deletions public/pages/workflow_detail/workflow_inputs/config_field_list.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,31 +55,6 @@ export function ConfigFieldList(props: ConfigFieldListProps) {
);
break;
}
// case 'map': {
// el = (
// <EuiFlexItem key={idx}>
// <MapField
// field={field}
// fieldPath={`${props.baseConfigPath}.${configId}.${field.id}`}
// onFormChange={props.onFormChange}
// />
// <EuiSpacer size={CONFIG_FIELD_SPACER_SIZE} />
// </EuiFlexItem>
// );
// break;
// }
// case 'json': {
// el = (
// <EuiFlexItem key={idx}>
// <JsonField
// label={field.label}
// placeholder={field.placeholder || ''}
// />
// <EuiSpacer size={INPUT_FIELD_SPACER_SIZE} />
// </EuiFlexItem>
// );
// break;
// }
}
return el;
})}
Expand Down
13 changes: 7 additions & 6 deletions public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,24 @@ interface MapFieldProps {
label: string;
helpLink?: string;
helpText?: string;
keyPlaceholder?: string;
valuePlaceholder?: string;
onFormChange: () => void;
}

/**
* Input component for configuring field mappings
*/
export function MapField(props: MapFieldProps) {
const { setFieldValue, errors, touched } = useFormikContext<
const { setFieldValue, setFieldTouched, errors, touched } = useFormikContext<
WorkflowFormValues
>();

// Adding a map entry to the end of the existing arr
function addMapEntry(curEntries: MapFormValue): void {
const updatedEntries = [...curEntries, { key: '', value: '' } as MapEntry];
setFieldValue(props.fieldPath, updatedEntries);
setFieldTouched(props.fieldPath, true);
props.onFormChange();
}

Expand All @@ -54,6 +57,7 @@ export function MapField(props: MapFieldProps) {
const updatedEntries = [...curEntries];
updatedEntries.splice(entryIndexToDelete, 1);
setFieldValue(props.fieldPath, updatedEntries);
setFieldTouched(props.fieldPath, true);
props.onFormChange();
}

Expand Down Expand Up @@ -97,10 +101,7 @@ export function MapField(props: MapFieldProps) {
startControl={
<input
type="string"
// TODO: find a way to config/title the placeholder text.
// For example, K/V values have different meanings if input
// map or output map for ML inference processors.
placeholder="Input"
placeholder={props.keyPlaceholder || 'Input'}
className="euiFieldText"
value={mapping.key}
onChange={(e) => {
Expand All @@ -119,7 +120,7 @@ export function MapField(props: MapFieldProps) {
endControl={
<input
type="string"
placeholder="Output"
placeholder={props.valuePlaceholder || 'Output'}
className="euiFieldText"
value={mapping.value}
onChange={(e) => {
Expand Down
128 changes: 108 additions & 20 deletions public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
*/

import React, { useState } from 'react';
import { useFormikContext } from 'formik';
import { useFormikContext, getIn } from 'formik';
import { isEmpty, get } from 'lodash';
import jsonpath from 'jsonpath';
import {
EuiButton,
EuiButtonEmpty,
EuiCodeBlock,
EuiCodeEditor,
EuiFlexGroup,
EuiFlexItem,
EuiModal,
Expand All @@ -21,8 +21,10 @@ import {
EuiText,
} from '@elastic/eui';
import {
IConfigField,
IProcessorConfig,
IngestPipelineConfig,
JSONPATH_ROOT_SELECTOR,
PROCESSOR_CONTEXT,
SimulateIngestPipelineDoc,
SimulateIngestPipelineResponse,
Expand All @@ -32,13 +34,16 @@ import {
import { formikToIngestPipeline, generateId } from '../../../../utils';
import { simulatePipeline, useAppDispatch } from '../../../../store';
import { getCore } from '../../../../services';
import { MapField } from '../input_fields';

interface InputTransformModalProps {
uiConfig: WorkflowConfig;
config: IProcessorConfig;
context: PROCESSOR_CONTEXT;
inputMapField: IConfigField;
inputMapFieldPath: string;
onClose: () => void;
onConfirm: () => void;
onFormChange: () => void;
}

/**
Expand All @@ -50,13 +55,16 @@ export function InputTransformModal(props: InputTransformModalProps) {

// source input / transformed output state
const [sourceInput, setSourceInput] = useState<string>('[]');
const [transformedOutput, setTransformedOutput] = useState<string>('TODO');
const [transformedOutput, setTransformedOutput] = useState<string>('[]');

// parse out the values and determine if there are none/some/all valid jsonpaths
const mapValues = getIn(values, `ingest.enrich.${props.config.id}.inputMap`);

return (
<EuiModal onClose={props.onClose} style={{ width: '70vw' }}>
<EuiModalHeader>
<EuiModalHeaderTitle>
<p>{`Configure input transform`}</p>
<p>{`Configure input`}</p>
</EuiModalHeaderTitle>
</EuiModalHeader>
<EuiModalBody>
Expand Down Expand Up @@ -116,24 +124,105 @@ export function InputTransformModal(props: InputTransformModalProps) {
</EuiFlexItem>
<EuiFlexItem>
<>
<EuiText>Define transform with JSONPath</EuiText>
<EuiText>Define transform</EuiText>
<EuiText size="s" color="subdued">
{`Dot notation is used by default. To explicitly use JSONPath, please ensure to prepend with the
root object selector "${JSONPATH_ROOT_SELECTOR}"`}
</EuiText>
<EuiSpacer size="s" />
<EuiCodeEditor
mode="json"
theme="textmate"
value={`TODO`}
readOnly={false}
setOptions={{
fontSize: '12px',
autoScrollEditorIntoView: true,
}}
tabSize={2}
<MapField
field={props.inputMapField}
fieldPath={props.inputMapFieldPath}
label="Input map"
helpText={`An array specifying how to map fields from the ingested document to the model’s input.`}
helpLink={
'https://opensearch.org/docs/latest/ingest-pipelines/processors/ml-inference/#configuration-parameters'
}
keyPlaceholder="Model input field"
valuePlaceholder="Document field"
onFormChange={props.onFormChange}
/>
</>
</EuiFlexItem>
<EuiFlexItem>
<>
<EuiText>Expected output</EuiText>
<EuiButton
style={{ width: '100px' }}
disabled={
isEmpty(mapValues) || isEmpty(JSON.parse(sourceInput))
}
onClick={async () => {
switch (props.context) {
case PROCESSOR_CONTEXT.INGEST: {
if (
!isEmpty(mapValues) &&
!isEmpty(JSON.parse(sourceInput))
) {
let output = {};
let sampleSourceInput = {};
try {
sampleSourceInput = JSON.parse(sourceInput)[0];
} catch {}

mapValues.forEach(
(mapValue: { key: string; value: string }) => {
const path = mapValue.value;
try {
let transformedResult = undefined;
// ML inference processors will use standard dot notation or JSONPath depending on the input.
// We follow the same logic here to generate consistent results.
if (
mapValue.value.startsWith(
JSONPATH_ROOT_SELECTOR
)
) {
// JSONPath transform
transformedResult = jsonpath.query(
sampleSourceInput,
path
);
// Bracket notation not supported - throw an error
} else if (
mapValue.value.includes(']') ||
mapValue.value.includes(']')
) {
throw new Error();
// Standard dot notation
} else {
transformedResult = get(
sampleSourceInput,
path
);
}

output = {
...output,
[mapValue.key]: transformedResult || '',
};

setTransformedOutput(
JSON.stringify(output, undefined, 2)
);
} catch (e: any) {
console.error(e);
getCore().notifications.toasts.addDanger(
'Error generating expected output. Ensure your inputs are valid JSONPath or dot notation syntax.',
e
);
}
}
);
}

break;
}
// TODO: complete for search request / search response contexts
}
}}
>
Generate
</EuiButton>
<EuiSpacer size="s" />
<EuiCodeBlock fontSize="m" isCopyable={false}>
{transformedOutput}
Expand All @@ -143,9 +232,8 @@ export function InputTransformModal(props: InputTransformModalProps) {
</EuiFlexGroup>
</EuiModalBody>
<EuiModalFooter>
<EuiButtonEmpty onClick={props.onClose}>Cancel</EuiButtonEmpty>
<EuiButton onClick={props.onConfirm} fill={true} color="primary">
Save
<EuiButton onClick={props.onClose} fill={false} color="primary">
Close
</EuiButton>
</EuiModalFooter>
</EuiModal>
Expand Down
Loading

0 comments on commit d490d15

Please sign in to comment.