From 033169607c980413cf90c041bd25bb37297ffaaf Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:25:58 -0700 Subject: [PATCH] [Console] Convert lib/mappings to TypeScript (#4008) (#4500) * Convert mappings.js to TypeScript Signed-off-by: Sirazh Gabdullin * Convert mappings.test.js to TypeScript Signed-off-by: Sirazh Gabdullin * Update CHANGELOG.md Signed-off-by: Sirazh Gabdullin * Add test for getTypes with multi-index mode Signed-off-by: Sirazh Gabdullin * type update Signed-off-by: Sirazh Gabdullin * update typing Signed-off-by: Sirazh Gabdullin * CHANGELOG fix Signed-off-by: Sirazh Gabdullin * Changelog update Signed-off-by: Sirazh Gabdullin * Update Changelog Signed-off-by: Sirazh Gabdullin --------- Signed-off-by: Sirazh Gabdullin Signed-off-by: Kawika Avilla Signed-off-by: Josh Romero Signed-off-by: Anan Zhuang Co-authored-by: Kawika Avilla Co-authored-by: Qingyang(Abby) Hu Co-authored-by: Ashwin P Chandran Co-authored-by: Anan Zhuang Co-authored-by: Josh Romero (cherry picked from commit e07c5e6646782440b5b0720fc45f3e39e3107cfd) Signed-off-by: github-actions[bot] # Conflicts: # CHANGELOG.md Co-authored-by: github-actions[bot] --- .../{mapping.test.js => mapping.test.ts} | 36 +++- .../lib/mappings/{mappings.js => mappings.ts} | 184 +++++++++++------- 2 files changed, 151 insertions(+), 69 deletions(-) rename src/plugins/console/public/lib/mappings/__tests__/{mapping.test.js => mapping.test.ts} (87%) rename src/plugins/console/public/lib/mappings/{mappings.js => mappings.ts} (66%) diff --git a/src/plugins/console/public/lib/mappings/__tests__/mapping.test.js b/src/plugins/console/public/lib/mappings/__tests__/mapping.test.ts similarity index 87% rename from src/plugins/console/public/lib/mappings/__tests__/mapping.test.js rename to src/plugins/console/public/lib/mappings/__tests__/mapping.test.ts index 4c0b88d21c0b..b5b1975a6a5c 100644 --- a/src/plugins/console/public/lib/mappings/__tests__/mapping.test.js +++ b/src/plugins/console/public/lib/mappings/__tests__/mapping.test.ts @@ -28,6 +28,7 @@ * under the License. */ +import { Field } from '../mappings'; import '../../../application/models/sense_editor/sense_editor.test.mocks'; import * as mappings from '../mappings'; @@ -39,7 +40,7 @@ describe('Mappings', () => { mappings.clear(); }); - function fc(f1, f2) { + function fc(f1: Field, f2: Field) { if (f1.name < f2.name) { return -1; } @@ -49,8 +50,8 @@ describe('Mappings', () => { return 0; } - function f(name, type) { - return { name: name, type: type || 'string' }; + function f(name: string, type?: string) { + return { name, type: type || 'string' }; } test('Multi fields 1.0 style', function () { @@ -256,10 +257,37 @@ describe('Mappings', () => { 'test_index2', ]); expect(mappings.getIndices(false).sort()).toEqual(['test_index1', 'test_index2']); - expect(mappings.expandAliases(['alias1', 'test_index2']).sort()).toEqual([ + expect((mappings.expandAliases(['alias1', 'test_index2']) as string[]).sort()).toEqual([ 'test_index1', 'test_index2', ]); expect(mappings.expandAliases('alias2')).toEqual('test_index2'); }); + + test('Multi types', function () { + mappings.loadMappings({ + index: { + properties: { + name1: { + type: 'object', + path: 'just_name', + properties: { + first1: { type: 'string' }, + last1: { type: 'string', index_name: 'i_last_1' }, + }, + }, + name2: { + type: 'object', + path: 'full', + properties: { + first2: { type: 'string' }, + last2: { type: 'string', index_name: 'i_last_2' }, + }, + }, + }, + }, + }); + + expect(mappings.getTypes()).toEqual(['properties']); + }); }); diff --git a/src/plugins/console/public/lib/mappings/mappings.js b/src/plugins/console/public/lib/mappings/mappings.ts similarity index 66% rename from src/plugins/console/public/lib/mappings/mappings.js rename to src/plugins/console/public/lib/mappings/mappings.ts index be5490ca1c1c..2bd425ade95e 100644 --- a/src/plugins/console/public/lib/mappings/mappings.js +++ b/src/plugins/console/public/lib/mappings/mappings.ts @@ -29,18 +29,65 @@ */ import _ from 'lodash'; +import { HttpResponse, HttpSetup } from 'opensearch-dashboards/public'; import * as opensearch from '../opensearch/opensearch'; +import { Settings } from '../../services'; + +export interface Field { + name: string; + type: string; +} + +interface FieldMapping { + enabled?: boolean; + path?: string; + properties?: Properties; + type?: string; + index_name?: string; + fields?: Record; + index?: string; +} + +type Properties = Record; + +interface IndexMapping { + [index: string]: { + [mapping: string]: FieldMapping | Properties; + }; +} + +interface IndexMappingOld { + [index: string]: { + mappings: { + properties: Properties; + }; + }; +} + +interface Aliases { + [aliasName: string]: Record; +} + +interface IndexAliases { + [indexName: string]: { + aliases: Aliases; + }; +} + +type IndicesOrAliases = string | string[] | null; // NOTE: If this value ever changes to be a few seconds or less, it might introduce flakiness // due to timing issues in our app.js tests. const POLL_INTERVAL = 60000; -let pollTimeoutId; +let pollTimeoutId: NodeJS.Timeout | null; -let perIndexTypes = {}; -let perAliasIndices = {}; -let templates = []; +let perIndexTypes: { [index: string]: { [type: string]: Field[] } } = {}; +let perAliasIndices: { [alias: string]: string[] } = {}; +let templates: string[] = []; -export function expandAliases(indicesOrAliases) { +export function expandAliases( + indicesOrAliases: IndicesOrAliases | undefined +): IndicesOrAliases | undefined { // takes a list of indices or aliases or a string which may be either and returns a list of indices // returns a list for multiple values or a string for a single. @@ -52,15 +99,15 @@ export function expandAliases(indicesOrAliases) { indicesOrAliases = [indicesOrAliases]; } - indicesOrAliases = indicesOrAliases.map((iOrA) => { + const mappedIndicesOrAliases = indicesOrAliases.map((iOrA) => { if (perAliasIndices[iOrA]) { return perAliasIndices[iOrA]; } return [iOrA]; }); - let ret = [].concat.apply([], indicesOrAliases); + let ret: string[] = ([] as string[]).concat(...mappedIndicesOrAliases); ret.sort(); - ret = ret.reduce((result, value, index, array) => { + ret = ret.reduce((result, value, index, array) => { const last = array[index - 1]; if (last !== value) { result.push(value); @@ -75,9 +122,9 @@ export function getTemplates() { return [...templates]; } -export function getFields(indices, types) { +export function getFields(indices?: IndicesOrAliases, types?: IndicesOrAliases): Field[] { // get fields for indices and types. Both can be a list, a string or null (meaning all). - let ret = []; + let ret: Array = []; indices = expandAliases(indices); if (typeof indices === 'string') { @@ -96,8 +143,6 @@ export function getFields(indices, types) { ret.push(fields); } }); - - ret = [].concat.apply([], ret); } } else { // multi index mode. @@ -106,16 +151,13 @@ export function getFields(indices, types) { ret.push(getFields(index, types)); } }); - ret = [].concat.apply([], ret); } - return _.uniqBy(ret, function (f) { - return f.name + ':' + f.type; - }); + return _.uniqBy(_.flatten(ret), (f: Field) => f.name + ':' + f.type); } -export function getTypes(indices) { - let ret = []; +export function getTypes(indices?: IndicesOrAliases) { + let ret: string[] = []; indices = expandAliases(indices); if (typeof indices === 'string') { const typeDict = perIndexTypes[indices]; @@ -137,17 +179,17 @@ export function getTypes(indices) { // multi index mode. Object.keys(perIndexTypes).forEach((index) => { if (!indices || indices.includes(index)) { - ret.push(getTypes(index)); + ret.push(...getTypes(index)); } }); - ret = [].concat.apply([], ret); + ret = ([] as string[]).concat.apply([], ret); } return _.uniq(ret); } -export function getIndices(includeAliases) { - const ret = []; +export function getIndices(includeAliases?: boolean) { + const ret: string[] = []; Object.keys(perIndexTypes).forEach((index) => { ret.push(index); }); @@ -160,13 +202,13 @@ export function getIndices(includeAliases) { return ret; } -function getFieldNamesFromFieldMapping(fieldName, fieldMapping) { +function getFieldNamesFromFieldMapping(fieldName: string, fieldMapping: FieldMapping) { if (fieldMapping.enabled === false) { return []; } let nestedFields; - function applyPathSettings(nestedFieldNames) { + function applyPathSettings(nestedFieldNames: Field[]) { const pathType = fieldMapping.path || 'full'; if (pathType === 'full') { return nestedFieldNames.map((f) => { @@ -185,16 +227,18 @@ function getFieldNamesFromFieldMapping(fieldName, fieldMapping) { const fieldType = fieldMapping.type; - const ret = { name: fieldName, type: fieldType }; + const ret: Field = { name: fieldName, type: fieldType as string }; if (fieldMapping.index_name) { ret.name = fieldMapping.index_name; } if (fieldMapping.fields) { - nestedFields = Object.entries(fieldMapping.fields).flatMap(([fieldName, fieldMapping]) => { - return getFieldNamesFromFieldMapping(fieldName, fieldMapping); - }); + nestedFields = Object.entries(fieldMapping.fields).flatMap( + ([nestedFieldName, nestedFieldMapping]) => { + return getFieldNamesFromFieldMapping(nestedFieldName, nestedFieldMapping); + } + ); nestedFields = applyPathSettings(nestedFields); nestedFields.unshift(ret); return nestedFields; @@ -203,7 +247,7 @@ function getFieldNamesFromFieldMapping(fieldName, fieldMapping) { return [ret]; } -function getFieldNamesFromProperties(properties = {}) { +function getFieldNamesFromProperties(properties: Properties = {}): Field[] { const fieldList = Object.entries(properties).flatMap(([fieldName, fieldMapping]) => { return getFieldNamesFromFieldMapping(fieldName, fieldMapping); }); @@ -218,11 +262,12 @@ function loadTemplates(templatesObject = {}) { templates = Object.keys(templatesObject); } -export function loadMappings(mappings) { +export function loadMappings(mappings: IndexMapping | IndexMappingOld | '{}') { perIndexTypes = {}; + if (mappings === '{}') return; Object.entries(mappings).forEach(([index, indexMapping]) => { - const normalizedIndexMappings = {}; + const normalizedIndexMappings: Record = {}; // Migrate 1.0.0 mappings. This format has changed, so we need to extract the underlying mapping. if (indexMapping.mappings && Object.keys(indexMapping).length === 1) { @@ -231,7 +276,7 @@ export function loadMappings(mappings) { Object.entries(indexMapping).forEach(([typeName, typeMapping]) => { if (typeName === 'properties') { - const fieldList = getFieldNamesFromProperties(typeMapping); + const fieldList = getFieldNamesFromProperties(typeMapping as Properties); normalizedIndexMappings[typeName] = fieldList; } else { normalizedIndexMappings[typeName] = []; @@ -241,13 +286,13 @@ export function loadMappings(mappings) { }); } -export function loadAliases(aliases) { +export function loadAliases(aliases: IndexAliases) { perAliasIndices = {}; - Object.entries(aliases).forEach(([index, omdexAliases]) => { + Object.entries(aliases).forEach(([index, indexAliases]) => { // verify we have an index defined. useful when mapping loading is disabled perIndexTypes[index] = perIndexTypes[index] || {}; - Object.keys(omdexAliases.aliases || {}).forEach((alias) => { + Object.keys(indexAliases.aliases || {}).forEach((alias) => { if (alias === index) { return; } // alias which is identical to index means no index. @@ -269,8 +314,13 @@ export function clear() { templates = []; } -function retrieveSettings(http, settingsKey, settingsToRetrieve, dataSourceId) { - const settingKeyToPathMap = { +function retrieveSettings( + http: HttpSetup, + settingsKey: string, + settingsToRetrieve: any, + dataSourceId: string +): Promise> | Promise | Promise<{}> { + const settingKeyToPathMap: { [settingsKey: string]: string } = { fields: '_mapping', indices: '_aliases', templates: '_template', @@ -307,17 +357,20 @@ export function clearSubscriptions() { } } -const retrieveMappings = async (http, settingsToRetrieve, dataSourceId) => { - const { body: mappings } = await retrieveSettings( - http, - 'fields', - settingsToRetrieve, - dataSourceId - ); - if (mappings) { +// Type guard function +function isHttpResponse(object: any): object is HttpResponse { + return object && typeof object === 'object' && 'body' in object; +} + +const retrieveMappings = async (http: HttpSetup, settingsToRetrieve: any, dataSourceId: string) => { + const response = await retrieveSettings(http, 'fields', settingsToRetrieve, dataSourceId); + + if (isHttpResponse(response) && response.body) { + const mappings = response.body; const maxMappingSize = Object.keys(mappings).length > 10 * 1024 * 1024; - let mappingsResponse; + let mappingsResponse: IndexMapping | IndexMappingOld | '{}'; if (maxMappingSize) { + // eslint-disable-next-line no-console console.warn( `Mapping size is larger than 10MB (${ Object.keys(mappings).length / 1024 / 1024 @@ -325,35 +378,31 @@ const retrieveMappings = async (http, settingsToRetrieve, dataSourceId) => { ); mappingsResponse = '{}'; } else { - mappingsResponse = mappings; + mappingsResponse = mappings as IndexMapping | IndexMappingOld; } loadMappings(mappingsResponse); } }; -const retrieveAliases = async (http, settingsToRetrieve, dataSourceId) => { - const { body: aliases } = await retrieveSettings( - http, - 'indices', - settingsToRetrieve, - dataSourceId - ); +const retrieveAliases = async (http: HttpSetup, settingsToRetrieve: any, dataSourceId: string) => { + const response = await retrieveSettings(http, 'fields', settingsToRetrieve, dataSourceId); - if (aliases) { + if (isHttpResponse(response) && response.body) { + const aliases = response.body as IndexAliases; loadAliases(aliases); } }; -const retrieveTemplates = async (http, settingsToRetrieve, dataSourceId) => { - const { body: templates } = await retrieveSettings( - http, - 'templates', - settingsToRetrieve, - dataSourceId - ); +const retrieveTemplates = async ( + http: HttpSetup, + settingsToRetrieve: any, + dataSourceId: string +) => { + const response = await retrieveSettings(http, 'fields', settingsToRetrieve, dataSourceId); - if (templates) { - loadTemplates(templates); + if (isHttpResponse(response) && response.body) { + const resTemplates = response.body; + loadTemplates(resTemplates); } }; @@ -362,7 +411,12 @@ const retrieveTemplates = async (http, settingsToRetrieve, dataSourceId) => { * @param settings Settings A way to retrieve the current settings * @param settingsToRetrieve any */ -export function retrieveAutoCompleteInfo(http, settings, settingsToRetrieve, dataSourceId) { +export function retrieveAutoCompleteInfo( + http: HttpSetup, + settings: Settings, + settingsToRetrieve: any, + dataSourceId: string +) { clearSubscriptions(); Promise.allSettled([