Merge branch 'main' into side_nav_1

Signed-off-by: Sai Medhini Reddy Maryada <[email protected]>

common/constants.ts

@@ -62,10 +62,13 @@ export enum WORKFLOW_TYPE {
   UNKNOWN = 'Unknown',
 }
 
+// the names should be consistent with the underlying implementation. used when generating the
+// final ingest/search pipeline configurations.
 export enum PROCESSOR_TYPE {
-  ML = 'ml_processor',
-  SPLIT = 'split_processor',
-  SORT = 'sort_processor',
+  ML = 'ml_inference',
+  SPLIT = 'split',
+  SORT = 'sort',
+  TEXT_CHUNKING = 'text_chunking',
 }
 
 export enum MODEL_TYPE {
@@ -118,6 +121,24 @@ export const ML_INFERENCE_DOCS_LINK =
   'https://opensearch.org/docs/latest/ingest-pipelines/processors/ml-inference/#configuration-parameters';
 export const ML_CHOOSE_MODEL_LINK =
   'https://opensearch.org/docs/latest/ml-commons-plugin/integrating-ml-models/#choosing-a-model';
+export const TEXT_CHUNKING_PROCESSOR_LINK =
+  'https://opensearch.org/docs/latest/ingest-pipelines/processors/text-chunking/';
+
+/**
+ * Text chunking algorithm constants
+ */
+export enum TEXT_CHUNKING_ALGORITHM {
+  FIXED_TOKEN_LENGTH = 'fixed_token_length',
+  DELIMITER = 'delimiter',
+}
+export const FIXED_TOKEN_LENGTH_OPTIONAL_FIELDS = [
+  'token_limit',
+  'tokenizer',
+  'overlap_rate',
+];
+export const DELIMITER_OPTIONAL_FIELDS = ['delimiter'];
+export const SHARED_OPTIONAL_FIELDS = ['max_chunk_limit', 'description', 'tag'];
+
 /**
  * MISCELLANEOUS
  */

common/interfaces.ts

@@ -16,23 +16,27 @@ export type Index = {
 /**
  ********** WORKFLOW TYPES/INTERFACES **********
  */
+export type MDSQueryParams = {
+  dataSourceId?: string;
+};
 
 export type ConfigFieldType =
   | 'string'
   | 'json'
   | 'jsonArray'
+  | 'jsonString'
   | 'select'
   | 'model'
   | 'map'
-  | 'mapArray';
+  | 'mapArray'
+  | 'boolean'
+  | 'number';
 
 export type ConfigFieldValue = string | {};
 
 export interface IConfigField {
   type: ConfigFieldType;
   id: string;
-  optional?: boolean;
-  label?: string;
   value?: ConfigFieldValue;
   selectOptions?: ConfigFieldValue[];
 }
@@ -41,6 +45,7 @@ export interface IConfig {
   id: string;
   name: string;
   fields: IConfigField[];
+  optionalFields?: IConfigField[];
 }
 
 export interface IProcessorConfig extends IConfig {
@@ -57,15 +62,25 @@ export type IndexConfig = {
   settings: IConfigField;
 };
 
+// TODO: may expand to just IndexConfig (including mappings/settings info)
+// if we want to persist this for users using some existing index,
+// and want to pass that index config around.
+export type SearchIndexConfig = {
+  name: IConfigField;
+};
+
 export type IngestConfig = {
-  enabled: boolean;
+  enabled: IConfigField;
   source: {};
+  pipelineName: IConfigField;
   enrich: ProcessorsConfig;
   index: IndexConfig;
 };
 
 export type SearchConfig = {
   request: IConfigField;
+  index: SearchIndexConfig;
+  pipelineName: IConfigField;
   enrichRequest: ProcessorsConfig;
   enrichResponse: ProcessorsConfig;
 };
@@ -207,6 +222,7 @@ export type MLInferenceProcessor = IngestProcessor & {
     model_id: string;
     input_map?: {};
     output_map?: {};
+    [key: string]: any;
   };
 };
 

common/utils.ts

@@ -5,7 +5,22 @@
 
 import moment from 'moment';
 import { DATE_FORMAT_PATTERN } from './';
+import { isEmpty } from 'lodash';
 
 export function toFormattedDate(timestampMillis: number): String {
   return moment(new Date(timestampMillis)).format(DATE_FORMAT_PATTERN);
 }
+
+const PERMISSIONS_ERROR_PATTERN = /no permissions for \[(.+)\] and User \[name=(.+), backend_roles/;
+
+export const prettifyErrorMessage = (rawErrorMessage: string) => {
+  if (isEmpty(rawErrorMessage) || rawErrorMessage === 'undefined') {
+    return 'Unknown error is returned.';
+  }
+  const match = rawErrorMessage.match(PERMISSIONS_ERROR_PATTERN);
+  if (isEmpty(match)) {
+    return rawErrorMessage;
+  } else {
+    return `User ${match[2]} has no permissions to [${match[1]}].`;
+  }
+};

opensearch_dashboards.json

@@ -9,5 +9,5 @@
     "navigation",
     "opensearchDashboardsUtils"
   ],
-  "optionalPlugins": []
+  "optionalPlugins": ["dataSource","dataSourceManagement"]
 }

public/app.tsx

@@ -4,7 +4,12 @@
  */
 
 import React from 'react';
-import { Route, RouteComponentProps, Switch } from 'react-router-dom';
+import {
+  Route,
+  RouteComponentProps,
+  Switch,
+  useLocation,
+} from 'react-router-dom';
 import {
   EuiPageSideBar,
   EuiSideNav,
@@ -18,16 +23,25 @@ import {
   WorkflowDetailRouterProps,
   WorkflowsRouterProps,
 } from './pages';
+import { MountPoint } from '../../../src/core/public';
+import {
+  constructHrefWithDataSourceId,
+  getDataSourceFromURL,
+} from './utils/utils';
 
 // styling
 import './global-styles.scss';
 
 interface Props extends RouteComponentProps {
+  setHeaderActionMenu: (menuMount?: MountPoint) => void;
   hideInAppSideNavBar: boolean;
 }
 
 export const FlowFrameworkDashboardsApp = (props: Props) => {
-  const { hideInAppSideNavBar } = props;
+  const { setHeaderActionMenu, hideInAppSideNavBar } = props;
+  const location = useLocation();
+  const queryParams = getDataSourceFromURL(location);
+  const dataSourceId = queryParams.dataSourceId;
   const sidebar = (
     <EuiPageSideBar
       style={{ minWidth: 190 }}
@@ -44,7 +58,10 @@ export const FlowFrameworkDashboardsApp = (props: Props) => {
               {
                 name: Navigation.Workflows,
                 id: 1,
-                href: `#${APP_PATH.WORKFLOWS}`,
+                href: constructHrefWithDataSourceId(
+                  APP_PATH.WORKFLOWS,
+                  dataSourceId
+                ),
                 isSelected: props.location.pathname === APP_PATH.WORKFLOWS,
               },
             ],
@@ -68,12 +85,17 @@ export const FlowFrameworkDashboardsApp = (props: Props) => {
             path={APP_PATH.WORKFLOW_DETAIL}
             render={(
               routeProps: RouteComponentProps<WorkflowDetailRouterProps>
-            ) => <WorkflowDetail {...routeProps} />}
+            ) => (
+              <WorkflowDetail
+                setActionMenu={setHeaderActionMenu}
+                {...routeProps}
+              />
+            )}
           />
           <Route
             path={APP_PATH.WORKFLOWS}
             render={(routeProps: RouteComponentProps<WorkflowsRouterProps>) => (
-              <Workflows {...routeProps} />
+              <Workflows setActionMenu={setHeaderActionMenu} {...routeProps} />
             )}
           />
           {/*
@@ -89,7 +111,12 @@ export const FlowFrameworkDashboardsApp = (props: Props) => {
                   pathname: APP_PATH.WORKFLOWS,
                 });
               }
-              return <Workflows {...routeProps} />;
+              return (
+                <Workflows
+                  setActionMenu={setHeaderActionMenu}
+                  {...routeProps}
+                />
+              );
             }}
           />
         </Switch>

public/configs/base_config.ts

@@ -12,14 +12,14 @@ export abstract class BaseConfig implements IConfig {
   id: string;
   name: string;
   fields: IConfigField[];
-  // TODO: have a dedicated optional fields list to display more fields & have more
-  // flexibility for the users to customize
+  optionalFields?: IConfigField[];
 
   // No-op constructor. If there are general / defaults for field values, add in here.
   constructor() {
     this.id = '';
     this.name = '';
     this.fields = [];
+    this.optionalFields = [];
   }
 
   // Persist a standard toObj() fn that all component classes can use. This is necessary
@@ -29,6 +29,7 @@ export abstract class BaseConfig implements IConfig {
       id: this.id,
       name: this.name,
       fields: this.fields,
+      optionalFields: this.optionalFields,
     } as IConfig;
   }
 }

public/configs/ingest_processors/index.ts

@@ -6,3 +6,4 @@
 export * from './ml_ingest_processor';
 export * from './split_ingest_processor';
 export * from './sort_ingest_processor';
+export * from './text_chunking_ingest_processor';

public/configs/ingest_processors/text_chunking_ingest_processor.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { PROCESSOR_TYPE, TEXT_CHUNKING_ALGORITHM } from '../../../common';
+import { generateId } from '../../utils';
+import { Processor } from '../processor';
+
+/**
+ * The text chunking ingest processor
+ */
+export class TextChunkingIngestProcessor extends Processor {
+  constructor() {
+    super();
+    this.name = 'Text Chunking Processor';
+    this.type = PROCESSOR_TYPE.TEXT_CHUNKING;
+    this.id = generateId('text_chunking_processor_ingest');
+    this.fields = [
+      {
+        id: 'field_map',
+        type: 'map',
+      },
+      {
+        id: 'algorithm',
+        type: 'select',
+        selectOptions: [
+          TEXT_CHUNKING_ALGORITHM.FIXED_TOKEN_LENGTH,
+          TEXT_CHUNKING_ALGORITHM.DELIMITER,
+        ],
+      },
+    ];
+    // optional params include all of those possible from both text chunking algorithms.
+    // for more details, see https://opensearch.org/docs/latest/ingest-pipelines/processors/text-chunking/
+    // the list of optional params per algorithm and shared across algorithms is persisted in
+    // common/constants.ts
+    this.optionalFields = [
+      // fixed_token_length optional params
+      {
+        id: 'token_limit',
+        type: 'number',
+        value: 384,
+      },
+      {
+        id: 'tokenizer',
+        type: 'string',
+        value: 'standard',
+      },
+      {
+        id: 'overlap_rate',
+        type: 'number',
+        value: 0,
+      },
+      // delimiter optional params
+      {
+        id: 'delimiter',
+        type: 'string',
+      },
+      // shared optional params (independent of algorithm)
+      {
+        id: 'max_chunk_limit',
+        type: 'number',
+        value: 100,
+      },
+      {
+        id: 'description',
+        type: 'string',
+      },
+      {
+        id: 'tag',
+        type: 'string',
+      },
+    ];
+  }
+}

public/configs/ml_processor.ts

@@ -21,13 +21,47 @@ export abstract class MLProcessor extends Processor {
         type: 'model',
       },
       {
-        id: 'inputMap',
+        id: 'input_map',
         type: 'mapArray',
       },
       {
-        id: 'outputMap',
+        id: 'output_map',
         type: 'mapArray',
       },
     ];
+    this.optionalFields = [
+      {
+        id: 'description',
+        type: 'string',
+      },
+      {
+        id: 'model_config',
+        type: 'json',
+      },
+      {
+        id: 'full_response_path',
+        type: 'boolean',
+        value: false,
+      },
+      {
+        id: 'ignore_missing',
+        type: 'boolean',
+        value: false,
+      },
+      {
+        id: 'ignore_failure',
+        type: 'boolean',
+        value: false,
+      },
+      {
+        id: 'max_prediction_tasks',
+        type: 'number',
+        value: 10,
+      },
+      {
+        id: 'tag',
+        type: 'string',
+      },
+    ];
   }
 }