[maps][file upload] remove number_of_shards index setting (#165390)

closes #165366 closes #165367 close #165697 replaces #165337 Serverless elasticsearch does not support index setting `number_of_shards` PR resolves issue be removing `number_of_shards`. When `number_of_shards` was introduced way back when, the default value was 5. Now the default value is one. Therefore there is no point providing the setting since its the same as the default. Just removing so code works across both serverless and traditional deployments PR also cleans up some types that are duplicative of elasticsearch types --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Tiago Costa <[email protected]>
elastic · Sep 6, 2023 · 2ec1218 · 2ec1218
1 parent 118ea87
commit 2ec1218
Show file tree

Hide file tree

Showing 13 changed files with 73 additions and 106 deletions.
diff --git a/...lugins/data_visualizer/public/application/common/components/combined_fields/utils.test.ts b/...lugins/data_visualizer/public/application/common/components/combined_fields/utils.test.ts
@@ -24,10 +24,10 @@ test('addCombinedFieldsToMappings', () => {
     },
     properties: {
       lat: {
-        type: 'number',
+        type: 'float' as 'float',
       },
       lon: {
-        type: 'number',
+        type: 'float' as 'float',
       },
     },
   };
@@ -37,10 +37,10 @@ test('addCombinedFieldsToMappings', () => {
     },
     properties: {
       lat: {
-        type: 'number',
+        type: 'float',
       },
       lon: {
-        type: 'number',
+        type: 'float',
       },
       location: {
         type: 'geo_point',
@@ -56,13 +56,13 @@ test('removeCombinedFieldsFromMappings', () => {
     },
     properties: {
       lat: {
-        type: 'number',
+        type: 'float' as 'float',
       },
       lon: {
-        type: 'number',
+        type: 'float' as 'float',
       },
       location: {
-        type: 'geo_point',
+        type: 'geo_point' as 'geo_point',
       },
     },
   };
@@ -72,10 +72,10 @@ test('removeCombinedFieldsFromMappings', () => {
     },
     properties: {
       lat: {
-        type: 'number',
+        type: 'float',
       },
       lon: {
-        type: 'number',
+        type: 'float',
       },
     },
   });

diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.ts b/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.ts
@@ -8,11 +8,8 @@
 import { i18n } from '@kbn/i18n';
 import { cloneDeep } from 'lodash';
 import { v4 as uuidv4 } from 'uuid';
-import type {
-  FindFileStructureResponse,
-  IngestPipeline,
-  Mappings,
-} from '@kbn/file-upload-plugin/common';
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { FindFileStructureResponse, IngestPipeline } from '@kbn/file-upload-plugin/common';
 import { CombinedField } from './types';
 
 const COMMON_LAT_NAMES = ['latitude', 'lat'];
@@ -28,23 +25,23 @@ export function getDefaultCombinedFields(results: FindFileStructureResponse) {
 }
 
 export function addCombinedFieldsToMappings(
-  mappings: Mappings,
+  mappings: MappingTypeMapping,
   combinedFields: CombinedField[]
-): Mappings {
-  const updatedMappings = { ...mappings };
+): MappingTypeMapping {
+  const updatedMappings = { properties: {}, ...mappings };
   combinedFields.forEach((combinedField) => {
     updatedMappings.properties[combinedField.combinedFieldName] = {
-      type: combinedField.mappingType,
+      type: combinedField.mappingType as any,
     };
   });
   return updatedMappings;
 }
 
 export function removeCombinedFieldsFromMappings(
-  mappings: Mappings,
+  mappings: MappingTypeMapping,
   combinedFields: CombinedField[]
 ) {
-  const updatedMappings = { ...mappings };
+  const updatedMappings = { properties: {}, ...mappings };
   combinedFields.forEach((combinedField) => {
     delete updatedMappings.properties[combinedField.combinedFieldName];
   });

diff --git a/..._visualizer/public/application/file_data_visualizer/components/import_view/import_view.js b/..._visualizer/public/application/file_data_visualizer/components/import_view/import_view.js
@@ -37,7 +37,7 @@ import {
 import { MODE as DATAVISUALIZER_MODE } from '../file_data_visualizer_view/constants';
 
 const DEFAULT_TIME_FIELD = '@timestamp';
-const DEFAULT_INDEX_SETTINGS = { number_of_shards: 1 };
+const DEFAULT_INDEX_SETTINGS = {};
 const CONFIG_MODE = { SIMPLE: 0, ADVANCED: 1 };
 
 const DEFAULT_STATE = {

diff --git a/x-pack/plugins/file_upload/common/index.ts b/x-pack/plugins/file_upload/common/index.ts
@@ -14,5 +14,4 @@ export type {
   FindFileStructureResponse,
   InputOverrides,
   IngestPipeline,
-  Mappings,
 } from './types';
diff --git a/x-pack/plugins/file_upload/common/types.ts b/x-pack/plugins/file_upload/common/types.ts
@@ -124,22 +124,6 @@ export interface ImportDocMessage {
 
 export type ImportDoc = ImportDocMessage | string | object;
 
-export interface Settings {
-  pipeline?: string;
-  index: string;
-  body: any[];
-  [key: string]: any;
-}
-
-export interface Mappings {
-  _meta?: {
-    created_by: string;
-  };
-  properties: {
-    [key: string]: any;
-  };
-}
-
 export interface IngestPipelineWrapper {
   id: string;
   pipeline: IngestPipeline;

diff --git a/x-pack/plugins/file_upload/public/components/geo_upload_wizard.tsx b/x-pack/plugins/file_upload/public/components/geo_upload_wizard.tsx
@@ -15,7 +15,6 @@ import { ImportCompleteView } from './import_complete_view';
 import type { FileUploadComponentProps, FileUploadGeoResults } from '../lazy_load_bundle';
 import { ImportResults } from '../importer';
 import { GeoFileImporter } from '../importer/geo';
-import type { Settings } from '../../common/types';
 import { hasImportPermission } from '../api';
 import { getPartialImportMessage } from './utils';
 
@@ -103,9 +102,6 @@ export class GeoUploadWizard extends Component<FileUploadComponentProps, State>
     //
     // create index
     //
-    const settings = {
-      number_of_shards: 1,
-    } as unknown as Settings;
     const mappings = {
       properties: {
         geometry: {
@@ -127,7 +123,7 @@ export class GeoUploadWizard extends Component<FileUploadComponentProps, State>
     this._geoFileImporter.setGeoFieldType(this.state.geoFieldType);
     const initializeImportResp = await this._geoFileImporter.initializeImport(
       this.state.indexName,
-      settings,
+      {},
       mappings,
       ingestPipeline
     );

diff --git a/x-pack/plugins/file_upload/public/importer/importer.ts b/x-pack/plugins/file_upload/public/importer/importer.ts
@@ -7,18 +7,15 @@
 
 import { chunk, intersection } from 'lodash';
 import moment from 'moment';
+import type {
+  IndicesIndexSettings,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { i18n } from '@kbn/i18n';
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 import { getHttp } from '../kibana_services';
 import { MB } from '../../common/constants';
-import type {
-  ImportDoc,
-  ImportFailure,
-  ImportResponse,
-  Mappings,
-  Settings,
-  IngestPipeline,
-} from '../../common/types';
+import type { ImportDoc, ImportFailure, ImportResponse, IngestPipeline } from '../../common/types';
 import { CreateDocsResponse, IImporter, ImportResults } from './types';
 
 const CHUNK_SIZE = 5000;
@@ -63,8 +60,8 @@ export abstract class Importer implements IImporter {
 
   public async initializeImport(
     index: string,
-    settings: Settings,
-    mappings: Mappings,
+    settings: IndicesIndexSettings,
+    mappings: MappingTypeMapping,
     pipeline: IngestPipeline
   ) {
     updatePipelineTimezone(pipeline);
@@ -279,8 +276,8 @@ export function callImportRoute({
   id: string | undefined;
   index: string;
   data: ImportDoc[];
-  settings: Settings | unknown;
-  mappings: Mappings | unknown;
+  settings: IndicesIndexSettings;
+  mappings: MappingTypeMapping;
   ingestPipeline: {
     id?: string;
     pipeline?: IngestPipeline;

diff --git a/x-pack/plugins/file_upload/public/importer/types.ts b/x-pack/plugins/file_upload/public/importer/types.ts
@@ -6,17 +6,15 @@
  */
 
 import type {
-  ImportFailure,
-  IngestPipeline,
-  ImportDoc,
-  ImportResponse,
-  Mappings,
-  Settings,
-} from '../../common/types';
+  IndicesIndexSettings,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+import type { ImportFailure, IngestPipeline, ImportDoc, ImportResponse } from '../../common/types';
 
 export interface ImportConfig {
-  settings: Settings;
-  mappings: Mappings;
+  settings: IndicesIndexSettings;
+  mappings: MappingTypeMapping;
   pipeline: IngestPipeline;
 }
 
@@ -44,8 +42,8 @@ export interface IImporter {
   read(data: ArrayBuffer): { success: boolean };
   initializeImport(
     index: string,
-    settings: Settings,
-    mappings: Mappings,
+    settings: IndicesIndexSettings,
+    mappings: MappingTypeMapping,
     pipeline: IngestPipeline
   ): Promise<ImportResponse>;
   import(

diff --git a/x-pack/plugins/file_upload/server/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts
@@ -6,22 +6,21 @@
  */
 
 import { IScopedClusterClient } from '@kbn/core/server';
+import type {
+  BulkRequest,
+  IndicesCreateRequest,
+  IndicesIndexSettings,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { INDEX_META_DATA_CREATED_BY } from '../common/constants';
-import {
-  ImportResponse,
-  ImportFailure,
-  InputData,
-  Settings,
-  Mappings,
-  IngestPipelineWrapper,
-} from '../common/types';
+import { ImportResponse, ImportFailure, InputData, IngestPipelineWrapper } from '../common/types';
 
 export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
   async function importData(
     id: string | undefined,
     index: string,
-    settings: Settings,
-    mappings: Mappings,
+    settings: IndicesIndexSettings,
+    mappings: MappingTypeMapping,
     ingestPipeline: IngestPipelineWrapper,
     data: InputData
   ): Promise<ImportResponse> {
@@ -89,8 +88,12 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
     }
   }
 
-  async function createIndex(index: string, settings: Settings, mappings: Mappings) {
-    const body: { mappings: Mappings; settings?: Settings } = {
+  async function createIndex(
+    index: string,
+    settings: IndicesIndexSettings,
+    mappings: MappingTypeMapping
+  ) {
+    const body: IndicesCreateRequest['body'] = {
       mappings: {
         _meta: {
           created_by: INDEX_META_DATA_CREATED_BY,
@@ -103,7 +106,6 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
       body.settings = settings;
     }
 
-    // @ts-expect-error settings.index is not compatible
     await asCurrentUser.indices.create({ index, body }, { maxRetries: 0 });
   }
 
@@ -115,12 +117,12 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
         body.push(data[i]);
       }
 
-      const settings: Settings = { index, body };
+      const bulkRequest: BulkRequest = { index, body };
       if (pipelineId !== undefined) {
-        settings.pipeline = pipelineId;
+        bulkRequest.pipeline = pipelineId;
       }
 
-      const resp = await asCurrentUser.bulk(settings, { maxRetries: 0 });
+      const resp = await asCurrentUser.bulk(bulkRequest, { maxRetries: 0 });
       if (resp.errors) {
         throw resp;
       } else {

diff --git a/x-pack/plugins/file_upload/server/routes.ts b/x-pack/plugins/file_upload/server/routes.ts
@@ -8,8 +8,12 @@
 import { schema } from '@kbn/config-schema';
 import { IScopedClusterClient } from '@kbn/core/server';
 import { CoreSetup, Logger } from '@kbn/core/server';
+import type {
+  IndicesIndexSettings,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { MAX_FILE_SIZE_BYTES } from '../common/constants';
-import type { IngestPipelineWrapper, InputData, Mappings, Settings } from '../common/types';
+import type { IngestPipelineWrapper, InputData } from '../common/types';
 import { wrapError } from './error_wrapper';
 import { importDataProvider } from './import_data';
 import { getTimeFieldRange } from './get_time_field_range';
@@ -29,8 +33,8 @@ function importData(
   client: IScopedClusterClient,
   id: string | undefined,
   index: string,
-  settings: Settings,
-  mappings: Mappings,
+  settings: IndicesIndexSettings,
+  mappings: MappingTypeMapping,
   ingestPipeline: IngestPipelineWrapper,
   data: InputData
 ) {

diff --git a/x-pack/plugins/maps/common/types.ts b/x-pack/plugins/maps/common/types.ts
@@ -17,19 +17,6 @@ export interface MatchingIndexesResp {
   error?: Error;
 }
 
-export interface IndexSourceMappings {
-  _meta?: {
-    created_by: string;
-  };
-  properties: {
-    [key: string]: any;
-  };
-}
-
-export interface BodySettings {
-  [key: string]: any;
-}
-
 export interface WriteSettings {
   index: string;
   body: object;

diff --git a/...ns/maps/public/classes/layers/wizards/new_vector_layer_wizard/create_new_index_pattern.ts b/...ns/maps/public/classes/layers/wizards/new_vector_layer_wizard/create_new_index_pattern.ts
@@ -5,16 +5,17 @@
  * 2.0.
  */
 
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { getHttp } from '../../../../kibana_services';
-import { CreateDocSourceResp, IndexSourceMappings } from '../../../../../common/types';
+import { CreateDocSourceResp } from '../../../../../common/types';
 import { INDEX_SOURCE_API_PATH } from '../../../../../common/constants';
 
 export const createNewIndexAndPattern = async ({
   indexName,
   defaultMappings = {},
 }: {
   indexName: string;
-  defaultMappings: IndexSourceMappings | {};
+  defaultMappings: MappingTypeMapping | {};
 }) => {
   return await getHttp().fetch<CreateDocSourceResp>({
     path: `/${INDEX_SOURCE_API_PATH}`,