From 2ec121850c433a526bf32d68e91cdcb0aa301417 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 6 Sep 2023 12:20:11 -0600 Subject: [PATCH] [maps][file upload] remove number_of_shards index setting (#165390) closes https://github.com/elastic/kibana/issues/165366 closes https://github.com/elastic/kibana/issues/165367 close https://github.com/elastic/kibana/issues/165697 replaces https://github.com/elastic/kibana/pull/165337 Serverless elasticsearch does not support index setting `number_of_shards` PR resolves issue be removing `number_of_shards`. When `number_of_shards` was introduced way back when, the default value was 5. Now the default value is one. Therefore there is no point providing the setting since its the same as the default. Just removing so code works across both serverless and traditional deployments PR also cleans up some types that are duplicative of elasticsearch types --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Tiago Costa --- .../components/combined_fields/utils.test.ts | 18 +++++----- .../components/combined_fields/utils.ts | 19 +++++------ .../components/import_view/import_view.js | 2 +- x-pack/plugins/file_upload/common/index.ts | 1 - x-pack/plugins/file_upload/common/types.ts | 16 --------- .../public/components/geo_upload_wizard.tsx | 6 +--- .../file_upload/public/importer/importer.ts | 21 +++++------- .../file_upload/public/importer/types.ts | 20 +++++------ .../plugins/file_upload/server/import_data.ts | 34 ++++++++++--------- x-pack/plugins/file_upload/server/routes.ts | 10 ++++-- x-pack/plugins/maps/common/types.ts | 13 ------- .../create_new_index_pattern.ts | 5 +-- .../server/data_indexing/create_doc_source.ts | 14 ++++---- 13 files changed, 73 insertions(+), 106 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.test.ts b/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.test.ts index 4bd84554eef60..30cda39cef9d2 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.test.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.test.ts @@ -24,10 +24,10 @@ test('addCombinedFieldsToMappings', () => { }, properties: { lat: { - type: 'number', + type: 'float' as 'float', }, lon: { - type: 'number', + type: 'float' as 'float', }, }, }; @@ -37,10 +37,10 @@ test('addCombinedFieldsToMappings', () => { }, properties: { lat: { - type: 'number', + type: 'float', }, lon: { - type: 'number', + type: 'float', }, location: { type: 'geo_point', @@ -56,13 +56,13 @@ test('removeCombinedFieldsFromMappings', () => { }, properties: { lat: { - type: 'number', + type: 'float' as 'float', }, lon: { - type: 'number', + type: 'float' as 'float', }, location: { - type: 'geo_point', + type: 'geo_point' as 'geo_point', }, }, }; @@ -72,10 +72,10 @@ test('removeCombinedFieldsFromMappings', () => { }, properties: { lat: { - type: 'number', + type: 'float', }, lon: { - type: 'number', + type: 'float', }, }, }); diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.ts b/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.ts index cd3bf01224e43..3910be982b2d0 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.ts @@ -8,11 +8,8 @@ import { i18n } from '@kbn/i18n'; import { cloneDeep } from 'lodash'; import { v4 as uuidv4 } from 'uuid'; -import type { - FindFileStructureResponse, - IngestPipeline, - Mappings, -} from '@kbn/file-upload-plugin/common'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { FindFileStructureResponse, IngestPipeline } from '@kbn/file-upload-plugin/common'; import { CombinedField } from './types'; const COMMON_LAT_NAMES = ['latitude', 'lat']; @@ -28,23 +25,23 @@ export function getDefaultCombinedFields(results: FindFileStructureResponse) { } export function addCombinedFieldsToMappings( - mappings: Mappings, + mappings: MappingTypeMapping, combinedFields: CombinedField[] -): Mappings { - const updatedMappings = { ...mappings }; +): MappingTypeMapping { + const updatedMappings = { properties: {}, ...mappings }; combinedFields.forEach((combinedField) => { updatedMappings.properties[combinedField.combinedFieldName] = { - type: combinedField.mappingType, + type: combinedField.mappingType as any, }; }); return updatedMappings; } export function removeCombinedFieldsFromMappings( - mappings: Mappings, + mappings: MappingTypeMapping, combinedFields: CombinedField[] ) { - const updatedMappings = { ...mappings }; + const updatedMappings = { properties: {}, ...mappings }; combinedFields.forEach((combinedField) => { delete updatedMappings.properties[combinedField.combinedFieldName]; }); diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js index db6929c7d4a66..0da5486e8d19d 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js @@ -37,7 +37,7 @@ import { import { MODE as DATAVISUALIZER_MODE } from '../file_data_visualizer_view/constants'; const DEFAULT_TIME_FIELD = '@timestamp'; -const DEFAULT_INDEX_SETTINGS = { number_of_shards: 1 }; +const DEFAULT_INDEX_SETTINGS = {}; const CONFIG_MODE = { SIMPLE: 0, ADVANCED: 1 }; const DEFAULT_STATE = { diff --git a/x-pack/plugins/file_upload/common/index.ts b/x-pack/plugins/file_upload/common/index.ts index 5d312835c3e06..eb5fcdc6b1c00 100644 --- a/x-pack/plugins/file_upload/common/index.ts +++ b/x-pack/plugins/file_upload/common/index.ts @@ -14,5 +14,4 @@ export type { FindFileStructureResponse, InputOverrides, IngestPipeline, - Mappings, } from './types'; diff --git a/x-pack/plugins/file_upload/common/types.ts b/x-pack/plugins/file_upload/common/types.ts index 46c3153e702ce..e8906bbfe469c 100644 --- a/x-pack/plugins/file_upload/common/types.ts +++ b/x-pack/plugins/file_upload/common/types.ts @@ -124,22 +124,6 @@ export interface ImportDocMessage { export type ImportDoc = ImportDocMessage | string | object; -export interface Settings { - pipeline?: string; - index: string; - body: any[]; - [key: string]: any; -} - -export interface Mappings { - _meta?: { - created_by: string; - }; - properties: { - [key: string]: any; - }; -} - export interface IngestPipelineWrapper { id: string; pipeline: IngestPipeline; diff --git a/x-pack/plugins/file_upload/public/components/geo_upload_wizard.tsx b/x-pack/plugins/file_upload/public/components/geo_upload_wizard.tsx index b3d1711b1d2a8..643642958e2b5 100644 --- a/x-pack/plugins/file_upload/public/components/geo_upload_wizard.tsx +++ b/x-pack/plugins/file_upload/public/components/geo_upload_wizard.tsx @@ -15,7 +15,6 @@ import { ImportCompleteView } from './import_complete_view'; import type { FileUploadComponentProps, FileUploadGeoResults } from '../lazy_load_bundle'; import { ImportResults } from '../importer'; import { GeoFileImporter } from '../importer/geo'; -import type { Settings } from '../../common/types'; import { hasImportPermission } from '../api'; import { getPartialImportMessage } from './utils'; @@ -103,9 +102,6 @@ export class GeoUploadWizard extends Component // // create index // - const settings = { - number_of_shards: 1, - } as unknown as Settings; const mappings = { properties: { geometry: { @@ -127,7 +123,7 @@ export class GeoUploadWizard extends Component this._geoFileImporter.setGeoFieldType(this.state.geoFieldType); const initializeImportResp = await this._geoFileImporter.initializeImport( this.state.indexName, - settings, + {}, mappings, ingestPipeline ); diff --git a/x-pack/plugins/file_upload/public/importer/importer.ts b/x-pack/plugins/file_upload/public/importer/importer.ts index 630c35aa794c7..14629e041082d 100644 --- a/x-pack/plugins/file_upload/public/importer/importer.ts +++ b/x-pack/plugins/file_upload/public/importer/importer.ts @@ -7,18 +7,15 @@ import { chunk, intersection } from 'lodash'; import moment from 'moment'; +import type { + IndicesIndexSettings, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { i18n } from '@kbn/i18n'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import { getHttp } from '../kibana_services'; import { MB } from '../../common/constants'; -import type { - ImportDoc, - ImportFailure, - ImportResponse, - Mappings, - Settings, - IngestPipeline, -} from '../../common/types'; +import type { ImportDoc, ImportFailure, ImportResponse, IngestPipeline } from '../../common/types'; import { CreateDocsResponse, IImporter, ImportResults } from './types'; const CHUNK_SIZE = 5000; @@ -63,8 +60,8 @@ export abstract class Importer implements IImporter { public async initializeImport( index: string, - settings: Settings, - mappings: Mappings, + settings: IndicesIndexSettings, + mappings: MappingTypeMapping, pipeline: IngestPipeline ) { updatePipelineTimezone(pipeline); @@ -279,8 +276,8 @@ export function callImportRoute({ id: string | undefined; index: string; data: ImportDoc[]; - settings: Settings | unknown; - mappings: Mappings | unknown; + settings: IndicesIndexSettings; + mappings: MappingTypeMapping; ingestPipeline: { id?: string; pipeline?: IngestPipeline; diff --git a/x-pack/plugins/file_upload/public/importer/types.ts b/x-pack/plugins/file_upload/public/importer/types.ts index 0ef8ace26f546..2a5efaa0d1dc9 100644 --- a/x-pack/plugins/file_upload/public/importer/types.ts +++ b/x-pack/plugins/file_upload/public/importer/types.ts @@ -6,17 +6,15 @@ */ import type { - ImportFailure, - IngestPipeline, - ImportDoc, - ImportResponse, - Mappings, - Settings, -} from '../../common/types'; + IndicesIndexSettings, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import type { ImportFailure, IngestPipeline, ImportDoc, ImportResponse } from '../../common/types'; export interface ImportConfig { - settings: Settings; - mappings: Mappings; + settings: IndicesIndexSettings; + mappings: MappingTypeMapping; pipeline: IngestPipeline; } @@ -44,8 +42,8 @@ export interface IImporter { read(data: ArrayBuffer): { success: boolean }; initializeImport( index: string, - settings: Settings, - mappings: Mappings, + settings: IndicesIndexSettings, + mappings: MappingTypeMapping, pipeline: IngestPipeline ): Promise; import( diff --git a/x-pack/plugins/file_upload/server/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts index f8555cbd7f1d3..198ad2afd5f8d 100644 --- a/x-pack/plugins/file_upload/server/import_data.ts +++ b/x-pack/plugins/file_upload/server/import_data.ts @@ -6,22 +6,21 @@ */ import { IScopedClusterClient } from '@kbn/core/server'; +import type { + BulkRequest, + IndicesCreateRequest, + IndicesIndexSettings, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { INDEX_META_DATA_CREATED_BY } from '../common/constants'; -import { - ImportResponse, - ImportFailure, - InputData, - Settings, - Mappings, - IngestPipelineWrapper, -} from '../common/types'; +import { ImportResponse, ImportFailure, InputData, IngestPipelineWrapper } from '../common/types'; export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { async function importData( id: string | undefined, index: string, - settings: Settings, - mappings: Mappings, + settings: IndicesIndexSettings, + mappings: MappingTypeMapping, ingestPipeline: IngestPipelineWrapper, data: InputData ): Promise { @@ -89,8 +88,12 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { } } - async function createIndex(index: string, settings: Settings, mappings: Mappings) { - const body: { mappings: Mappings; settings?: Settings } = { + async function createIndex( + index: string, + settings: IndicesIndexSettings, + mappings: MappingTypeMapping + ) { + const body: IndicesCreateRequest['body'] = { mappings: { _meta: { created_by: INDEX_META_DATA_CREATED_BY, @@ -103,7 +106,6 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { body.settings = settings; } - // @ts-expect-error settings.index is not compatible await asCurrentUser.indices.create({ index, body }, { maxRetries: 0 }); } @@ -115,12 +117,12 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { body.push(data[i]); } - const settings: Settings = { index, body }; + const bulkRequest: BulkRequest = { index, body }; if (pipelineId !== undefined) { - settings.pipeline = pipelineId; + bulkRequest.pipeline = pipelineId; } - const resp = await asCurrentUser.bulk(settings, { maxRetries: 0 }); + const resp = await asCurrentUser.bulk(bulkRequest, { maxRetries: 0 }); if (resp.errors) { throw resp; } else { diff --git a/x-pack/plugins/file_upload/server/routes.ts b/x-pack/plugins/file_upload/server/routes.ts index 0fbcbd413c238..707314b5bd4a2 100644 --- a/x-pack/plugins/file_upload/server/routes.ts +++ b/x-pack/plugins/file_upload/server/routes.ts @@ -8,8 +8,12 @@ import { schema } from '@kbn/config-schema'; import { IScopedClusterClient } from '@kbn/core/server'; import { CoreSetup, Logger } from '@kbn/core/server'; +import type { + IndicesIndexSettings, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { MAX_FILE_SIZE_BYTES } from '../common/constants'; -import type { IngestPipelineWrapper, InputData, Mappings, Settings } from '../common/types'; +import type { IngestPipelineWrapper, InputData } from '../common/types'; import { wrapError } from './error_wrapper'; import { importDataProvider } from './import_data'; import { getTimeFieldRange } from './get_time_field_range'; @@ -29,8 +33,8 @@ function importData( client: IScopedClusterClient, id: string | undefined, index: string, - settings: Settings, - mappings: Mappings, + settings: IndicesIndexSettings, + mappings: MappingTypeMapping, ingestPipeline: IngestPipelineWrapper, data: InputData ) { diff --git a/x-pack/plugins/maps/common/types.ts b/x-pack/plugins/maps/common/types.ts index 5b202fd7811e5..76af2ac979404 100644 --- a/x-pack/plugins/maps/common/types.ts +++ b/x-pack/plugins/maps/common/types.ts @@ -17,19 +17,6 @@ export interface MatchingIndexesResp { error?: Error; } -export interface IndexSourceMappings { - _meta?: { - created_by: string; - }; - properties: { - [key: string]: any; - }; -} - -export interface BodySettings { - [key: string]: any; -} - export interface WriteSettings { index: string; body: object; diff --git a/x-pack/plugins/maps/public/classes/layers/wizards/new_vector_layer_wizard/create_new_index_pattern.ts b/x-pack/plugins/maps/public/classes/layers/wizards/new_vector_layer_wizard/create_new_index_pattern.ts index 90e0bceec7ef6..31762e7ef0375 100644 --- a/x-pack/plugins/maps/public/classes/layers/wizards/new_vector_layer_wizard/create_new_index_pattern.ts +++ b/x-pack/plugins/maps/public/classes/layers/wizards/new_vector_layer_wizard/create_new_index_pattern.ts @@ -5,8 +5,9 @@ * 2.0. */ +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { getHttp } from '../../../../kibana_services'; -import { CreateDocSourceResp, IndexSourceMappings } from '../../../../../common/types'; +import { CreateDocSourceResp } from '../../../../../common/types'; import { INDEX_SOURCE_API_PATH } from '../../../../../common/constants'; export const createNewIndexAndPattern = async ({ @@ -14,7 +15,7 @@ export const createNewIndexAndPattern = async ({ defaultMappings = {}, }: { indexName: string; - defaultMappings: IndexSourceMappings | {}; + defaultMappings: MappingTypeMapping | {}; }) => { return await getHttp().fetch({ path: `/${INDEX_SOURCE_API_PATH}`, diff --git a/x-pack/plugins/maps/server/data_indexing/create_doc_source.ts b/x-pack/plugins/maps/server/data_indexing/create_doc_source.ts index 435e56a8b60a9..12dc618a96578 100644 --- a/x-pack/plugins/maps/server/data_indexing/create_doc_source.ts +++ b/x-pack/plugins/maps/server/data_indexing/create_doc_source.ts @@ -5,12 +5,15 @@ * 2.0. */ +import type { + IndicesCreateRequest, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { ElasticsearchClient, IScopedClusterClient } from '@kbn/core/server'; import { DataViewsCommonService } from '@kbn/data-plugin/server'; -import { CreateDocSourceResp, IndexSourceMappings, BodySettings } from '../../common/types'; +import { CreateDocSourceResp } from '../../common/types'; import { MAPS_NEW_VECTOR_LAYER_META_CREATED_BY } from '../../common/constants'; -const DEFAULT_SETTINGS = { number_of_shards: 1 }; const DEFAULT_META = { _meta: { created_by: MAPS_NEW_VECTOR_LAYER_META_CREATED_BY, @@ -19,7 +22,7 @@ const DEFAULT_META = { export async function createDocSource( index: string, - mappings: IndexSourceMappings, + mappings: MappingTypeMapping, { asCurrentUser }: IScopedClusterClient, indexPatternsService: DataViewsCommonService ): Promise { @@ -41,15 +44,14 @@ export async function createDocSource( async function createIndex( indexName: string, - mappings: IndexSourceMappings, + mappings: MappingTypeMapping, asCurrentUser: ElasticsearchClient ) { - const body: { mappings: IndexSourceMappings; settings: BodySettings } = { + const body: IndicesCreateRequest['body'] = { mappings: { ...DEFAULT_META, ...mappings, }, - settings: DEFAULT_SETTINGS, }; await asCurrentUser.indices.create({ index: indexName, body });