Merge remote-tracking branch 'origin/2.x' into search-source-poc

src/core/server/core_route_handler_context.ts

@@ -41,7 +41,6 @@ import {
 import { Auditor } from './audit_trail';
 import { InternalUiSettingsServiceStart, IUiSettingsClient } from './ui_settings';
 import { InternalOpenSearchDataServiceStart } from './opensearch_data/types';
-import { ConnectionError } from '@opensearch-project/opensearch/lib/errors';
 
 class CoreOpenSearchRouteHandlerContext {
   #client?: IScopedClusterClient;

src/core/server/opensearch_data/client/data_source_client.ts

@@ -10,6 +10,7 @@ import { Client } from '@opensearch-project/opensearch';
 import { Logger } from '../../logging';
 import { OpenSearchClient, OpenSearchClientConfig } from '../../opensearch/client';
 import { SavedObjectsClientContract } from '../../saved_objects/types';
+// @ts-ignore
 import { CryptoCli } from '../../../../../src/plugins/credential_management/server/crypto/cli/crypto_cli';
 
 /**
@@ -54,10 +55,13 @@ interface CredentialInfo {
 export class DataSourceClient implements ICustomDataSourceClient {
   public dataSourceClientsPool: Map<string, Client>;
   private savedObjectClient: SavedObjectsClientContract;
-  private isDataSourceFeautureEnabled = true;
+  private isDataSourceFeatureEnabled = true;
   private isClosed = false;
+  // TODO: need add size limit(maps to config item in osd.yml)
+  private CLIENT_POOL_SIZE_LIMIT = 50;
 
   constructor(
+    // @ts-ignore
     private readonly config: OpenSearchClientConfig,
     savedObjectClient: SavedObjectsClientContract,
     logger: Logger
@@ -69,6 +73,9 @@ export class DataSourceClient implements ICustomDataSourceClient {
     // 2. throw error if isDataSourceEnabled == false, while API is called
   }
   async asDataSource(dataSourceId: string) {
+    if (!this.isDataSourceFeatureEnabled) {
+      throw Error('isDataSourceFeatureEnabled == false, please enable the feature in osd.yml');
+    }
     const { endpoint, credentialId } = await this.getDataSourceInfo(dataSourceId);
     const { username, password } = await this.getCredentialInfo(credentialId);
     // 2. build/find client and return
@@ -83,7 +90,7 @@ export class DataSourceClient implements ICustomDataSourceClient {
         },
       });
       // update pool
-      this.dataSourceClientsPool.set(dataSourceId, dataSourceClient);
+      this.updateDataSourceClientPool(dataSourceId, dataSourceClient);
     }
     return dataSourceClient;
   }
@@ -99,6 +106,15 @@ export class DataSourceClient implements ICustomDataSourceClient {
     return { endpoint, credentialId };
   }
 
+  private updateDataSourceClientPool(dataSourceId: string, dataSourceClient: Client) {
+    // TODO: need to optimize to pop up least recent used client instead of first one
+    if (this.dataSourceClientsPool.size > this.CLIENT_POOL_SIZE_LIMIT) {
+      const [firstDataSourceId] = this.dataSourceClientsPool.keys();
+      this.dataSourceClientsPool.delete(firstDataSourceId);
+      this.dataSourceClientsPool.set(dataSourceId, dataSourceClient);
+    }
+  }
+
   private async getCredentialInfo(credentialId: string): Promise<CredentialInfo> {
     /**
      * TODO:

src/core/server/opensearch_data/opensearch_data_service.ts

@@ -7,14 +7,13 @@
  */
 
 import { Observable, Subject } from 'rxjs';
-import { first, map, shareReplay, takeUntil } from 'rxjs/operators';
+import { first, map } from 'rxjs/operators';
 import { CoreService } from 'src/core/types';
 import { AuditorFactory, AuditTrailStart } from '../audit_trail';
 import { CoreContext } from '../core_context';
 import { Logger } from '../logging';
 import { OpenSearchClientConfig } from '../opensearch/client';
 import { OpenSearchConfig, OpenSearchConfigType } from '../opensearch/opensearch_config';
-import { pollOpenSearchNodesVersion } from '../opensearch/version_check/ensure_opensearch_version';
 import { InternalSavedObjectsServiceStart, SavedObjectsClient } from '../saved_objects';
 import { SavedObjectsClientContract } from '../types';
 import { DataSourceClient } from './client/data_source_client';
@@ -29,8 +28,10 @@ export class OpenSearchDataService
   implements CoreService<InternalOpenSearchDataServiceSetup, InternalOpenSearchDataServiceStart> {
   private readonly log: Logger;
   private readonly config$: Observable<OpenSearchConfig>;
+  //@ts-ignore
   private auditorFactory?: AuditorFactory;
   private stop$ = new Subject();
+  //@ts-ignore
   private opensearchDashboardsVersion: string;
   private savedObjectClient?: SavedObjectsClientContract;
   private dataSourceClient?: DataSourceClient;
@@ -46,7 +47,7 @@ export class OpenSearchDataService
   public async setup(): Promise<any> {
     this.log.debug('Setting up opensearch data service');
 
-    const config = await this.config$.pipe(first()).toPromise();
+    // const config = await this.config$.pipe(first()).toPromise(); //TODO: add later
 
     // TODO: update accordingly when we decide how to check node/version compatibility in setup stage
 

src/core/tsconfig.json

@@ -16,7 +16,10 @@
     "index.ts",
     "typings.ts"
   ],
-  "references": [
-    { "path": "../test_utils/" }
-  ]
+  "exclude": [
+    "server/**/*.test.ts",
+    "server/saved_objects/routes/test_utils.ts",
+    "server/test_utils.ts"
+  ],
+  "references": [{ "path": "../test_utils/" }]
 }

src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts

@@ -43,7 +43,13 @@ import { IndexPatternField, IIndexPatternFieldList, fieldList } from '../fields'
 import { formatHitProvider } from './format_hit';
 import { flattenHitWrapper } from './flatten_hit';
 import { FieldFormatsStartCommon, FieldFormat } from '../../field_formats';
-import { IndexPatternSpec, TypeMeta, SourceFilter, IndexPatternFieldMap } from '../types';
+import {
+  IndexPatternSpec,
+  TypeMeta,
+  SourceFilter,
+  IndexPatternFieldMap,
+  DataSourceRef,
+} from '../types';
 import { SerializedFieldFormat } from '../../../../expressions/common';
 
 interface IndexPatternDeps {
@@ -86,6 +92,9 @@ export class IndexPattern implements IIndexPattern {
   // savedObject version
   public version: string | undefined;
   public sourceFilters?: SourceFilter[];
+  public dataSourcesJSON?: string; // todo: cleanup, only keep one
+  public dataSourceRef?: DataSourceRef;
+
   private originalSavedObjectBody: SavedObjectBody = {};
   private shortDotsEnable: boolean = false;
   private fieldFormats: FieldFormatsStartCommon;
@@ -129,9 +138,20 @@ export class IndexPattern implements IIndexPattern {
     this.fieldFormatMap = _.mapValues(fieldFormatMap, (mapping) => {
       return this.deserializeFieldFormatMap(mapping);
     });
-    this.dataSourceId = spec.dataSourceId;
+
+    this.dataSourcesJSON = spec.dataSourcesJSON;
+    this.dataSourceRef = this.getDataSourceRef(spec.dataSourcesJSON);
   }
 
+  getDataSourceRef = (dataSourcesJSON?: string) => {
+    if (dataSourcesJSON) {
+      const refs = JSON.parse(dataSourcesJSON);
+      if (!_.isEmpty(refs)) {
+        return refs[0]; // only support 1-1 mapping now
+      }
+    }
+  };
+
   /**
    * Get last saved saved object fields
    */
@@ -356,6 +376,7 @@ export class IndexPattern implements IIndexPattern {
       fieldFormatMap,
       type: this.type,
       typeMeta: this.typeMeta ? JSON.stringify(this.typeMeta) : undefined,
+      dataSourcesJSON: this.dataSourcesJSON, // todo: maybe stringfy here instead
     };
   }
 

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -29,6 +29,8 @@
  */
 
 import { i18n } from '@osd/i18n';
+// eslint-disable-next-line @osd/eslint/no-restricted-paths
+import { IndexPatternSavedObject } from 'src/plugins/data/public/index_patterns';
 import { SavedObjectsClientCommon } from '../..';
 
 import { createIndexPatternCache } from '.';
@@ -56,6 +58,8 @@ import { SavedObjectNotFound } from '../../../../opensearch_dashboards_utils/com
 import { IndexPatternMissingIndices } from '../lib';
 import { findByTitle } from '../utils';
 import { DuplicateIndexPatternError } from '../errors';
+// eslint-disable-next-line @osd/eslint/no-restricted-paths
+import { SavedObjectLoader } from '../../../../saved_objects/public';
 
 const indexPatternCache = createIndexPatternCache();
 const MAX_ATTEMPTS_TO_RESOLVE_CONFLICTS = 3;
@@ -67,6 +71,7 @@ export interface IndexPatternSavedObjectAttrs {
 
 interface IndexPatternsServiceDeps {
   uiSettings: UiSettingsCommon;
+  savedIndexPattern: SavedObjectLoader;
   savedObjectsClient: SavedObjectsClientCommon;
   apiClient: IIndexPatternsApiClient;
   fieldFormats: FieldFormatsStartCommon;
@@ -78,6 +83,7 @@ interface IndexPatternsServiceDeps {
 
 export class IndexPatternsService {
   private config: UiSettingsCommon;
+  private savedIndexPattern: SavedObjectLoader;
   private savedObjectsClient: SavedObjectsClientCommon;
   private savedObjectsCache?: Array<SavedObject<IndexPatternSavedObjectAttrs>> | null;
   private apiClient: IIndexPatternsApiClient;
@@ -89,6 +95,7 @@ export class IndexPatternsService {
 
   constructor({
     uiSettings,
+    savedIndexPattern,
     savedObjectsClient,
     apiClient,
     fieldFormats,
@@ -108,6 +115,7 @@ export class IndexPatternsService {
       uiSettings,
       onRedirectNoIndexPattern
     );
+    this.savedIndexPattern = savedIndexPattern;
   }
 
   /**
@@ -119,7 +127,7 @@ export class IndexPatternsService {
       fields: ['title'],
       perPage: 10000,
     });
-  }
+  } // todo: does this find method imcompatible with using loader?
 
   /**
    * Get list of index pattern ids
@@ -378,6 +386,46 @@ export class IndexPatternsService {
     };
   };
 
+  /**
+   * Converts index pattern saved object to index pattern spec
+   * @param savedObject
+   */
+
+  indexPatternSavedObjectToSpec = (savedObject: IndexPatternSavedObject): IndexPatternSpec => {
+    const {
+      id,
+      version,
+      title,
+      timeFieldName,
+      intervalName,
+      fields,
+      sourceFilters,
+      fieldFormatMap,
+      typeMeta,
+      type,
+      dataSourcesJSON,
+    } = savedObject;
+
+    const parsedSourceFilters = sourceFilters ? JSON.parse(sourceFilters) : undefined;
+    const parsedTypeMeta = typeMeta ? JSON.parse(typeMeta) : undefined;
+    const parsedFieldFormatMap = fieldFormatMap ? JSON.parse(fieldFormatMap) : {};
+    const parsedFields: FieldSpec[] = fields ? JSON.parse(fields) : [];
+
+    this.addFormatsToFields(parsedFields, parsedFieldFormatMap);
+    return {
+      id,
+      version,
+      title,
+      intervalName,
+      timeFieldName,
+      sourceFilters: parsedSourceFilters,
+      fields: this.fieldArrayToMap(parsedFields),
+      typeMeta: parsedTypeMeta,
+      type,
+      dataSourcesJSON,
+    };
+  };
+
   /**
    * Get an index pattern by id. Cache optimized
    * @param id
@@ -389,24 +437,26 @@ export class IndexPatternsService {
       return cache;
     }
 
-    const savedObject = await this.savedObjectsClient.get<IndexPatternAttributes>(
-      savedObjectType,
-      id
-    );
+    // const savedObject = await this.savedObjectsClient.get<IndexPatternAttributes>(
+    //   savedObjectType,
+    //   id
+    // );
 
-    if (!savedObject.version) {
+    const savedIndexPattern: IndexPatternSavedObject = await this.savedIndexPattern.get(id);
+    const { version, fieldFormatMap } = savedIndexPattern;
+
+    if (!version) {
       throw new SavedObjectNotFound(
         savedObjectType,
         id,
         'management/opensearch-dashboards/indexPatterns'
       );
     }
 
-    const spec = this.savedObjectToSpec(savedObject);
+    // const spec = this.savedObjectToSpec(savedObject);
+    const spec = this.indexPatternSavedObjectToSpec(savedIndexPattern);
     const { title, type, typeMeta } = spec;
-    const parsedFieldFormats: FieldFormatMap = savedObject.attributes.fieldFormatMap
-      ? JSON.parse(savedObject.attributes.fieldFormatMap)
-      : {};
+    const parsedFieldFormats: FieldFormatMap = fieldFormatMap ? JSON.parse(fieldFormatMap) : {};
 
     const isFieldRefreshRequired = this.isFieldRefreshRequired(spec.fields);
     let isSaveRequired = isFieldRefreshRequired;
@@ -564,14 +614,34 @@ export class IndexPatternsService {
     }
 
     const body = indexPattern.getAsSavedObjectBody();
-    const response = await this.savedObjectsClient.create(savedObjectType, body, {
-      id: indexPattern.id,
-    });
-    indexPattern.id = response.id;
+    // todo: replace this
+    const savedIndexPattern: IndexPatternSavedObject = await this.savedIndexPattern.get();
+
+    this.populateSavedIndexPattern(savedIndexPattern, body);
+
+    // const response = await this.savedObjectsClient.create(savedObjectType, body, {
+    //   id: indexPattern.id, //todo: where is this id come from, cannot be the dup one
+    // });
+
+    // todo: 1. this will cause duplidate overrwrite as it defaults to true
+    const response = await savedIndexPattern.save({});
+    indexPattern.id = response; // response.id
     indexPatternCache.set(indexPattern.id, indexPattern);
     return indexPattern;
   }
 
+  populateSavedIndexPattern(emptyIndexPattern: IndexPatternSavedObject, body: Record<string, any>) {
+    emptyIndexPattern.title = body.title;
+    emptyIndexPattern.timeFieldName = body.timeFieldName;
+    emptyIndexPattern.intervalName = body.intervalName;
+    emptyIndexPattern.sourceFilters = body.sourceFilters;
+    emptyIndexPattern.fields = body.fields;
+    emptyIndexPattern.fieldFormatMap = body.fieldFormatMap;
+    emptyIndexPattern.type = body.type;
+    emptyIndexPattern.typeMeta = body.typeMeta;
+    emptyIndexPattern.dataSourcesJSON = body.dataSourcesJSON;
+  }
+
   /**
    * Save existing index pattern. Will attempt to merge differences if there are conflicts
    * @param indexPattern

src/plugins/data/common/index_patterns/types.ts

@@ -49,7 +49,7 @@ export interface IIndexPattern {
   getFormatterForField?: (
     field: IndexPatternField | IndexPatternField['spec'] | IFieldType
   ) => FieldFormat;
-  dataSourceId?: string;
+  dataSourceRef?: DataSourceRef;
 }
 
 export interface IndexPatternAttributes {
@@ -61,7 +61,6 @@ export interface IndexPatternAttributes {
   intervalName?: string;
   sourceFilters?: string;
   fieldFormatMap?: string;
-  dataSourceId?: string;
 }
 
 export type OnNotification = (toastInputFields: ToastInputFields) => void;
@@ -196,9 +195,14 @@ export interface IndexPatternSpec {
   fields?: IndexPatternFieldMap;
   typeMeta?: TypeMeta;
   type?: string;
-  dataSourceId?: string;
+  dataSourcesJSON?: string; // todo: refactor to dataSourceRefs : DataSourceRef[]
 }
 
 export interface SourceFilter {
   value: string;
 }
+
+export interface DataSourceRef {
+  id: string;
+  type: string;
+}

src/plugins/data/common/search/opensearch_search/types.ts

@@ -43,10 +43,6 @@ export interface ISearchOptions {
    * Use this option to force using a specific server side search strategy. Leave empty to use the default strategy.
    */
   strategy?: string;
-  /**
-   * Use this option to attache a data source id. Leave empty if the request is submitted to default OpenSearch cluster
-   */
-  dataSourceId?: string;
 }
 
 export type ISearchRequestParams<T = Record<string, any>> = {