diff --git a/.eslintrc.js b/.eslintrc.js index 9b00135df5bac..087d6276cd33f 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -82,12 +82,6 @@ module.exports = { 'react-hooks/exhaustive-deps': 'off', }, }, - { - files: ['src/legacy/core_plugins/vis_type_table/**/*.{js,ts,tsx}'], - rules: { - 'react-hooks/exhaustive-deps': 'off', - }, - }, { files: [ 'src/legacy/core_plugins/vis_default_editor/public/components/controls/**/*.{ts,tsx}', diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts index 51fa19c140bf0..e671061b34352 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts @@ -57,9 +57,28 @@ interface UpdateOptions extends IndexOptions { id: string; } +interface MigrateResponse { + success: boolean; + result: Array<{ status: string }>; +} + export class KbnClientSavedObjects { constructor(private readonly log: ToolingLog, private readonly requester: KbnClientRequester) {} + /** + * Run the saved objects migration + */ + public async migrate() { + this.log.debug('Migrating saved objects'); + + return await this.requester.request({ + description: 'migrate saved objects', + path: uriencode`/internal/saved_objects/_migrate`, + method: 'POST', + body: {}, + }); + } + /** * Get an object */ diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index 451db9750ada7..fe0491870e4bd 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -43784,6 +43784,18 @@ class KbnClientSavedObjects { this.log = log; this.requester = requester; } + /** + * Run the saved objects migration + */ + async migrate() { + this.log.debug('Migrating saved objects'); + return await this.requester.request({ + description: 'migrate saved objects', + path: kbn_client_requester_1.uriencode `/internal/saved_objects/_migrate`, + method: 'POST', + body: {}, + }); + } /** * Get an object */ diff --git a/src/core/server/saved_objects/migrations/core/build_active_mappings.test.ts b/src/core/server/saved_objects/migrations/core/build_active_mappings.test.ts index 821a10353f8ec..9d220cfdf94b7 100644 --- a/src/core/server/saved_objects/migrations/core/build_active_mappings.test.ts +++ b/src/core/server/saved_objects/migrations/core/build_active_mappings.test.ts @@ -21,6 +21,11 @@ import { IndexMapping } from './../../mappings'; import { buildActiveMappings, diffMappings } from './build_active_mappings'; describe('buildActiveMappings', () => { + test('creates a strict mapping', () => { + const mappings = buildActiveMappings({}); + expect(mappings.dynamic).toEqual('strict'); + }); + test('combines all mappings and includes core mappings', () => { const properties = { aaa: { type: 'text' }, diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index 494f834717def..bc29061b380b8 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -88,15 +88,27 @@ export class KibanaMigrator { } /** - * Migrates the mappings and documents in the Kibana index. This will run only + * Migrates the mappings and documents in the Kibana index. By default, this will run only * once and subsequent calls will return the result of the original call. * + * @param rerun - If true, method will run a new migration when called again instead of + * returning the result of the initial migration. This should only be used when factors external + * to Kibana itself alter the kibana index causing the saved objects mappings or data to change + * after the Kibana server performed the initial migration. + * + * @remarks When the `rerun` parameter is set to true, no checks are performed to ensure that no migration + * is currently running. Chained or concurrent calls to `runMigrations({ rerun: true })` can lead to + * multiple migrations running at the same time. When calling with this parameter, it's expected that the calling + * code should ensure that the initial call resolves before calling the function again. + * * @returns - A promise which resolves once all migrations have been applied. * The promise resolves with an array of migration statuses, one for each * elasticsearch index which was migrated. */ - public runMigrations(): Promise> { - if (this.migrationResult === undefined) { + public runMigrations({ rerun = false }: { rerun?: boolean } = {}): Promise< + Array<{ status: string }> + > { + if (this.migrationResult === undefined || rerun) { this.migrationResult = this.runMigrationsInternal(); } diff --git a/src/core/server/saved_objects/routes/index.ts b/src/core/server/saved_objects/routes/index.ts index f2f57798dd5f0..0afa24b18760b 100644 --- a/src/core/server/saved_objects/routes/index.ts +++ b/src/core/server/saved_objects/routes/index.ts @@ -20,6 +20,7 @@ import { InternalHttpServiceSetup } from '../../http'; import { Logger } from '../../logging'; import { SavedObjectConfig } from '../saved_objects_config'; +import { IKibanaMigrator } from '../migrations'; import { registerGetRoute } from './get'; import { registerCreateRoute } from './create'; import { registerDeleteRoute } from './delete'; @@ -32,17 +33,20 @@ import { registerLogLegacyImportRoute } from './log_legacy_import'; import { registerExportRoute } from './export'; import { registerImportRoute } from './import'; import { registerResolveImportErrorsRoute } from './resolve_import_errors'; +import { registerMigrateRoute } from './migrate'; export function registerRoutes({ http, logger, config, importableExportableTypes, + migratorPromise, }: { http: InternalHttpServiceSetup; logger: Logger; config: SavedObjectConfig; importableExportableTypes: string[]; + migratorPromise: Promise; }) { const router = http.createRouter('/api/saved_objects/'); @@ -58,4 +62,8 @@ export function registerRoutes({ registerExportRoute(router, config, importableExportableTypes); registerImportRoute(router, config, importableExportableTypes); registerResolveImportErrorsRoute(router, config, importableExportableTypes); + + const internalRouter = http.createRouter('/internal/saved_objects/'); + + registerMigrateRoute(internalRouter, migratorPromise); } diff --git a/src/core/server/saved_objects/routes/integration_tests/import.test.ts b/src/core/server/saved_objects/routes/integration_tests/import.test.ts index 2c8d568b750c6..954e6d9e4831a 100644 --- a/src/core/server/saved_objects/routes/integration_tests/import.test.ts +++ b/src/core/server/saved_objects/routes/integration_tests/import.test.ts @@ -32,7 +32,7 @@ const config = { maxImportExportSize: 10000, } as SavedObjectConfig; -describe('POST /api/saved_objects/_import', () => { +describe('POST /internal/saved_objects/_import', () => { let server: setupServerReturn['server']; let httpSetup: setupServerReturn['httpSetup']; let handlerContext: setupServerReturn['handlerContext']; @@ -51,7 +51,7 @@ describe('POST /api/saved_objects/_import', () => { savedObjectsClient.find.mockResolvedValue(emptyResponse); - const router = httpSetup.createRouter('/api/saved_objects/'); + const router = httpSetup.createRouter('/internal/saved_objects/'); registerImportRoute(router, config, allowedTypes); await server.start(); @@ -63,7 +63,7 @@ describe('POST /api/saved_objects/_import', () => { it('formats successful response', async () => { const result = await supertest(httpSetup.server.listener) - .post('/api/saved_objects/_import') + .post('/internal/saved_objects/_import') .set('content-Type', 'multipart/form-data; boundary=BOUNDARY') .send( [ @@ -99,7 +99,7 @@ describe('POST /api/saved_objects/_import', () => { }); const result = await supertest(httpSetup.server.listener) - .post('/api/saved_objects/_import') + .post('/internal/saved_objects/_import') .set('content-Type', 'multipart/form-data; boundary=EXAMPLE') .send( [ @@ -148,7 +148,7 @@ describe('POST /api/saved_objects/_import', () => { }); const result = await supertest(httpSetup.server.listener) - .post('/api/saved_objects/_import') + .post('/internal/saved_objects/_import') .set('content-Type', 'multipart/form-data; boundary=EXAMPLE') .send( [ @@ -199,7 +199,7 @@ describe('POST /api/saved_objects/_import', () => { }); const result = await supertest(httpSetup.server.listener) - .post('/api/saved_objects/_import') + .post('/internal/saved_objects/_import') .set('content-Type', 'multipart/form-data; boundary=EXAMPLE') .send( [ @@ -249,7 +249,7 @@ describe('POST /api/saved_objects/_import', () => { }); const result = await supertest(httpSetup.server.listener) - .post('/api/saved_objects/_import') + .post('/internal/saved_objects/_import') .set('content-Type', 'multipart/form-data; boundary=EXAMPLE') .send( [ diff --git a/src/core/server/saved_objects/routes/integration_tests/migrate.test.mocks.ts b/src/core/server/saved_objects/routes/integration_tests/migrate.test.mocks.ts new file mode 100644 index 0000000000000..870d50426904f --- /dev/null +++ b/src/core/server/saved_objects/routes/integration_tests/migrate.test.mocks.ts @@ -0,0 +1,26 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { mockKibanaMigrator } from '../../migrations/kibana/kibana_migrator.mock'; + +export const migratorInstanceMock = mockKibanaMigrator.create(); +export const KibanaMigratorMock = jest.fn().mockImplementation(() => migratorInstanceMock); +jest.doMock('../../migrations/kibana/kibana_migrator', () => ({ + KibanaMigrator: KibanaMigratorMock, +})); diff --git a/src/core/server/saved_objects/routes/integration_tests/migrate.test.ts b/src/core/server/saved_objects/routes/integration_tests/migrate.test.ts new file mode 100644 index 0000000000000..928d17e7e5be2 --- /dev/null +++ b/src/core/server/saved_objects/routes/integration_tests/migrate.test.ts @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { migratorInstanceMock } from './migrate.test.mocks'; +import * as kbnTestServer from '../../../../../test_utils/kbn_server'; + +describe('SavedObjects /_migrate endpoint', () => { + let root: ReturnType; + + beforeEach(async () => { + root = kbnTestServer.createRoot({ migrations: { skip: true } }); + await root.setup(); + await root.start(); + migratorInstanceMock.runMigrations.mockClear(); + }, 30000); + + afterEach(async () => { + await root.shutdown(); + }); + + it('calls runMigrations on the migrator with rerun=true when accessed', async () => { + await kbnTestServer.request + .post(root, '/internal/saved_objects/_migrate') + .send({}) + .expect(200); + + expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(1); + expect(migratorInstanceMock.runMigrations).toHaveBeenCalledWith({ rerun: true }); + }); + + it('calls runMigrations multiple time when multiple access', async () => { + await kbnTestServer.request + .post(root, '/internal/saved_objects/_migrate') + .send({}) + .expect(200); + + expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(1); + + await kbnTestServer.request + .post(root, '/internal/saved_objects/_migrate') + .send({}) + .expect(200); + + expect(migratorInstanceMock.runMigrations).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/core/server/saved_objects/routes/migrate.ts b/src/core/server/saved_objects/routes/migrate.ts new file mode 100644 index 0000000000000..69e99d10acd09 --- /dev/null +++ b/src/core/server/saved_objects/routes/migrate.ts @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { IRouter } from '../../http'; +import { IKibanaMigrator } from '../migrations'; + +export const registerMigrateRoute = ( + router: IRouter, + migratorPromise: Promise +) => { + router.post( + { + path: '/_migrate', + validate: false, + options: { + tags: ['access:migrateSavedObjects'], + }, + }, + router.handleLegacyErrors(async (context, req, res) => { + const migrator = await migratorPromise; + await migrator.runMigrations({ rerun: true }); + return res.ok({ + body: { + success: true, + }, + }); + }) + ); +}; diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index ece00539536e1..fa2b67a3e43b2 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -17,8 +17,9 @@ * under the License. */ -import { CoreService } from 'src/core/types'; +import { Subject } from 'rxjs'; import { first, filter, take } from 'rxjs/operators'; +import { CoreService } from '../../types'; import { SavedObjectsClient, SavedObjectsClientProvider, @@ -36,7 +37,7 @@ import { SavedObjectsMigrationConfigType, SavedObjectConfig, } from './saved_objects_config'; -import { InternalHttpServiceSetup, KibanaRequest } from '../http'; +import { KibanaRequest, InternalHttpServiceSetup } from '../http'; import { SavedObjectsClientContract, SavedObjectsType, SavedObjectsLegacyUiExports } from './types'; import { ISavedObjectsRepository, SavedObjectsRepository } from './service/lib/repository'; import { @@ -47,8 +48,8 @@ import { Logger } from '../logging'; import { convertLegacyTypes } from './utils'; import { SavedObjectTypeRegistry, ISavedObjectTypeRegistry } from './saved_objects_type_registry'; import { PropertyValidators } from './validation'; -import { registerRoutes } from './routes'; import { SavedObjectsSerializer } from './serialization'; +import { registerRoutes } from './routes'; /** * Saved Objects is Kibana's data persistence mechanism allowing plugins to @@ -201,9 +202,9 @@ export interface SavedObjectsRepositoryFactory { /** @internal */ export interface SavedObjectsSetupDeps { + http: InternalHttpServiceSetup; legacyPlugins: LegacyServiceDiscoverPlugins; elasticsearch: InternalElasticsearchServiceSetup; - http: InternalHttpServiceSetup; } interface WrappedClientFactoryWrapper { @@ -225,6 +226,7 @@ export class SavedObjectsService private clientFactoryProvider?: SavedObjectsClientFactoryProvider; private clientFactoryWrappers: WrappedClientFactoryWrapper[] = []; + private migrator$ = new Subject(); private typeRegistry = new SavedObjectTypeRegistry(); private validations: PropertyValidators = {}; @@ -262,6 +264,7 @@ export class SavedObjectsService http: setupDeps.http, logger: this.logger, config: this.config, + migratorPromise: this.migrator$.pipe(first()).toPromise(), importableExportableTypes, }); @@ -302,6 +305,8 @@ export class SavedObjectsService const adminClient = this.setupDeps!.elasticsearch.adminClient; const migrator = this.createMigrator(kibanaConfig, this.config.migration, migrationsRetryDelay); + this.migrator$.next(migrator); + /** * Note: We want to ensure that migrations have completed before * continuing with further Core start steps that might use SavedObjects diff --git a/src/es_archiver/actions/empty_kibana_index.ts b/src/es_archiver/actions/empty_kibana_index.ts index 5f96fbc5f996c..d61d544deadc4 100644 --- a/src/es_archiver/actions/empty_kibana_index.ts +++ b/src/es_archiver/actions/empty_kibana_index.ts @@ -32,9 +32,8 @@ export async function emptyKibanaIndexAction({ kbnClient: KbnClient; }) { const stats = createStats('emptyKibanaIndex', log); - const kibanaPluginIds = await kbnClient.plugins.getEnabledIds(); await deleteKibanaIndices({ client, stats, log }); - await migrateKibanaIndex({ client, log, kibanaPluginIds }); + await migrateKibanaIndex({ client, kbnClient }); return stats; } diff --git a/src/es_archiver/actions/load.ts b/src/es_archiver/actions/load.ts index 404fd0daea91d..ae7799205b299 100644 --- a/src/es_archiver/actions/load.ts +++ b/src/es_archiver/actions/load.ts @@ -106,7 +106,7 @@ export async function loadAction({ // If we affected the Kibana index, we need to ensure it's migrated... if (Object.keys(result).some(k => k.startsWith('.kibana'))) { - await migrateKibanaIndex({ client, log, kibanaPluginIds }); + await migrateKibanaIndex({ client, kbnClient }); if (kibanaPluginIds.includes('spaces')) { await createDefaultSpace({ client, index: '.kibana' }); diff --git a/src/es_archiver/lib/indices/kibana_index.ts b/src/es_archiver/lib/indices/kibana_index.ts index 56be86a23eda0..1867f24d6f9ed 100644 --- a/src/es_archiver/lib/indices/kibana_index.ts +++ b/src/es_archiver/lib/indices/kibana_index.ts @@ -17,42 +17,10 @@ * under the License. */ -import { get } from 'lodash'; -import fs from 'fs'; -import Path from 'path'; -import { promisify } from 'util'; -import { toArray } from 'rxjs/operators'; import { Client, CreateDocumentParams } from 'elasticsearch'; -import { ToolingLog } from '@kbn/dev-utils'; - +import { ToolingLog, KbnClient } from '@kbn/dev-utils'; import { Stats } from '../stats'; import { deleteIndex } from './delete_index'; -import { KibanaMigrator } from '../../../core/server/saved_objects/migrations'; -import { LegacyConfig } from '../../../core/server'; -import { convertLegacyTypes } from '../../../core/server/saved_objects/utils'; -import { SavedObjectTypeRegistry } from '../../../core/server/saved_objects'; -// @ts-ignore -import { collectUiExports } from '../../../legacy/ui/ui_exports'; -// @ts-ignore -import { findPluginSpecs } from '../../../legacy/plugin_discovery'; - -/** - * Load the uiExports for a Kibana instance, only load uiExports from xpack if - * it is enabled in the Kibana server. - */ -const getUiExports = async (kibanaPluginIds: string[]) => { - const xpackEnabled = kibanaPluginIds.includes('xpack_main'); - - const { spec$ } = await findPluginSpecs({ - plugins: { - scanDirs: [Path.resolve(__dirname, '../../../legacy/core_plugins')], - paths: xpackEnabled ? [Path.resolve(__dirname, '../../../../x-pack')] : [], - }, - }); - - const specs = await spec$.pipe(toArray()).toPromise(); - return collectUiExports(specs); -}; /** * Deletes all indices that start with `.kibana` @@ -93,61 +61,21 @@ export async function deleteKibanaIndices({ */ export async function migrateKibanaIndex({ client, - log, - kibanaPluginIds, + kbnClient, }: { client: Client; - log: ToolingLog; - kibanaPluginIds: string[]; + kbnClient: KbnClient; }) { - const uiExports = await getUiExports(kibanaPluginIds); - const kibanaVersion = await loadKibanaVersion(); - - const configKeys: Record = { - 'xpack.task_manager.index': '.kibana_task_manager', - }; - const config = { get: (path: string) => configKeys[path] }; - - const savedObjectTypes = convertLegacyTypes(uiExports, config as LegacyConfig); - const typeRegistry = new SavedObjectTypeRegistry(); - savedObjectTypes.forEach(type => typeRegistry.registerType(type)); - - const logger = { - trace: log.verbose.bind(log), - debug: log.debug.bind(log), - info: log.info.bind(log), - warn: log.warning.bind(log), - error: log.error.bind(log), - fatal: log.error.bind(log), - log: (entry: any) => log.info(entry.message), - get: () => logger, - }; - - const migratorOptions = { - savedObjectsConfig: { - scrollDuration: '5m', - batchSize: 100, - pollInterval: 100, - skip: false, + // we allow dynamic mappings on the index, as some interceptors are accessing documents before + // the migration is actually performed. The migrator will put the value back to `strict` after migration. + await client.indices.putMapping({ + index: '.kibana', + body: { + dynamic: true, }, - kibanaConfig: { - index: '.kibana', - } as any, - logger, - kibanaVersion, - typeRegistry, - savedObjectValidations: uiExports.savedObjectValidations, - callCluster: (path: string, ...args: any[]) => - (get(client, path) as Function).call(client, ...args), - }; - - return await new KibanaMigrator(migratorOptions).runMigrations(); -} + } as any); -async function loadKibanaVersion() { - const readFile = promisify(fs.readFile); - const packageJson = await readFile(Path.join(__dirname, '../../../../package.json')); - return JSON.parse(packageJson.toString('utf-8')).version; + return await kbnClient.savedObjects.migrate(); } /** diff --git a/src/legacy/core_plugins/data/public/actions/filters/brush_event.js b/src/legacy/core_plugins/data/public/actions/filters/brush_event.js deleted file mode 100644 index 67711bd4599a2..0000000000000 --- a/src/legacy/core_plugins/data/public/actions/filters/brush_event.js +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import _ from 'lodash'; -import moment from 'moment'; -import { esFilters } from '../../../../../../plugins/data/public'; -import { deserializeAggConfig } from '../../search/expressions/utils'; - -export async function onBrushEvent(event, getIndexPatterns) { - const isNumber = event.data.ordered; - const isDate = isNumber && event.data.ordered.date; - - const xRaw = _.get(event.data, 'series[0].values[0].xRaw'); - if (!xRaw) return []; - const column = xRaw.table.columns[xRaw.column]; - if (!column) return []; - if (!column.meta) return []; - const indexPattern = await getIndexPatterns().get(column.meta.indexPatternId); - const aggConfig = deserializeAggConfig({ - ...column.meta, - indexPattern, - }); - const field = aggConfig.params.field; - if (!field) return []; - - if (event.range.length <= 1) return []; - - const min = event.range[0]; - const max = event.range[event.range.length - 1]; - if (min === max) return []; - - let range; - - if (isDate) { - range = { - gte: moment(min).toISOString(), - lt: moment(max).toISOString(), - format: 'strict_date_optional_time', - }; - } else { - range = { - gte: min, - lt: max, - }; - } - - const newFilter = esFilters.buildRangeFilter( - field, - range, - indexPattern, - event.data.xAxisFormatter - ); - - return [newFilter]; -} diff --git a/src/legacy/core_plugins/data/public/actions/filters/brush_event.test.js b/src/legacy/core_plugins/data/public/actions/filters/brush_event.test.js deleted file mode 100644 index 743f6caee4edd..0000000000000 --- a/src/legacy/core_plugins/data/public/actions/filters/brush_event.test.js +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import _ from 'lodash'; -import moment from 'moment'; -import expect from '@kbn/expect'; - -jest.mock('../../search/aggs', () => ({ - AggConfigs: function AggConfigs() { - return { - createAggConfig: ({ params }) => ({ - params, - getIndexPattern: () => ({ - timeFieldName: 'time', - }), - }), - }; - }, -})); - -import { onBrushEvent } from './brush_event'; - -describe('brushEvent', () => { - const DAY_IN_MS = 24 * 60 * 60 * 1000; - const JAN_01_2014 = 1388559600000; - - const aggConfigs = [ - { - params: {}, - getIndexPattern: () => ({ - timeFieldName: 'time', - }), - }, - ]; - - const baseEvent = { - data: { - fieldFormatter: _.constant({}), - series: [ - { - values: [ - { - xRaw: { - column: 0, - table: { - columns: [ - { - id: '1', - meta: { - type: 'histogram', - indexPatternId: 'indexPatternId', - aggConfigParams: aggConfigs[0].params, - }, - }, - ], - }, - }, - }, - ], - }, - ], - }, - }; - - beforeEach(() => { - baseEvent.data.indexPattern = { - id: 'logstash-*', - timeFieldName: 'time', - }; - }); - - test('should be a function', () => { - expect(onBrushEvent).to.be.a(Function); - }); - - test('ignores event when data.xAxisField not provided', async () => { - const event = _.cloneDeep(baseEvent); - const filters = await onBrushEvent(event, () => ({ - get: () => baseEvent.data.indexPattern, - })); - expect(filters.length).to.equal(0); - }); - - describe('handles an event when the x-axis field is a date field', () => { - describe('date field is index pattern timefield', () => { - let dateEvent; - const dateField = { - name: 'time', - type: 'date', - }; - - beforeEach(() => { - aggConfigs[0].params.field = dateField; - dateEvent = _.cloneDeep(baseEvent); - dateEvent.data.ordered = { date: true }; - }); - - test('by ignoring the event when range spans zero time', async () => { - const event = _.cloneDeep(dateEvent); - event.range = [JAN_01_2014, JAN_01_2014]; - const filters = await onBrushEvent(event, () => ({ - get: () => dateEvent.data.indexPattern, - })); - expect(filters.length).to.equal(0); - }); - - test('by updating the timefilter', async () => { - const event = _.cloneDeep(dateEvent); - event.range = [JAN_01_2014, JAN_01_2014 + DAY_IN_MS]; - const filters = await onBrushEvent(event, () => ({ - get: async () => dateEvent.data.indexPattern, - })); - expect(filters[0].range.time.gte).to.be(new Date(JAN_01_2014).toISOString()); - // Set to a baseline timezone for comparison. - expect(filters[0].range.time.lt).to.be(new Date(JAN_01_2014 + DAY_IN_MS).toISOString()); - }); - }); - - describe('date field is not index pattern timefield', () => { - let dateEvent; - const dateField = { - name: 'anotherTimeField', - type: 'date', - }; - - beforeEach(() => { - aggConfigs[0].params.field = dateField; - dateEvent = _.cloneDeep(baseEvent); - dateEvent.data.ordered = { date: true }; - }); - - test('creates a new range filter', async () => { - const event = _.cloneDeep(dateEvent); - const rangeBegin = JAN_01_2014; - const rangeEnd = rangeBegin + DAY_IN_MS; - event.range = [rangeBegin, rangeEnd]; - const filters = await onBrushEvent(event, () => ({ - get: () => dateEvent.data.indexPattern, - })); - expect(filters.length).to.equal(1); - expect(filters[0].range.anotherTimeField.gte).to.equal(moment(rangeBegin).toISOString()); - expect(filters[0].range.anotherTimeField.lt).to.equal(moment(rangeEnd).toISOString()); - expect(filters[0].range.anotherTimeField).to.have.property('format'); - expect(filters[0].range.anotherTimeField.format).to.equal('strict_date_optional_time'); - }); - }); - }); - - describe('handles an event when the x-axis field is a number', () => { - let numberEvent; - const numberField = { - name: 'numberField', - type: 'number', - }; - - beforeEach(() => { - aggConfigs[0].params.field = numberField; - numberEvent = _.cloneDeep(baseEvent); - numberEvent.data.ordered = { date: false }; - }); - - test('by ignoring the event when range does not span at least 2 values', async () => { - const event = _.cloneDeep(numberEvent); - event.range = [1]; - const filters = await onBrushEvent(event, () => ({ - get: () => numberEvent.data.indexPattern, - })); - expect(filters.length).to.equal(0); - }); - - test('by creating a new filter', async () => { - const event = _.cloneDeep(numberEvent); - event.range = [1, 2, 3, 4]; - const filters = await onBrushEvent(event, () => ({ - get: () => numberEvent.data.indexPattern, - })); - expect(filters.length).to.equal(1); - expect(filters[0].range.numberField.gte).to.equal(1); - expect(filters[0].range.numberField.lt).to.equal(4); - expect(filters[0].range.numberField).not.to.have.property('format'); - }); - }); -}); diff --git a/src/legacy/core_plugins/data/public/actions/filters/brush_event.test.ts b/src/legacy/core_plugins/data/public/actions/filters/brush_event.test.ts new file mode 100644 index 0000000000000..0e18c7c707fa3 --- /dev/null +++ b/src/legacy/core_plugins/data/public/actions/filters/brush_event.test.ts @@ -0,0 +1,208 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import moment from 'moment'; + +jest.mock('../../search/aggs', () => ({ + AggConfigs: function AggConfigs() { + return { + createAggConfig: ({ params }: Record) => ({ + params, + getIndexPattern: () => ({ + timeFieldName: 'time', + }), + }), + }; + }, +})); + +jest.mock('../../../../../../plugins/data/public/services', () => ({ + getIndexPatterns: () => { + return { + get: async () => { + return { + id: 'logstash-*', + timeFieldName: 'time', + }; + }, + }; + }, +})); + +import { onBrushEvent, BrushEvent } from './brush_event'; + +describe('brushEvent', () => { + const DAY_IN_MS = 24 * 60 * 60 * 1000; + const JAN_01_2014 = 1388559600000; + let baseEvent: BrushEvent; + + const aggConfigs = [ + { + params: { + field: {}, + }, + getIndexPattern: () => ({ + timeFieldName: 'time', + }), + }, + ]; + + beforeEach(() => { + baseEvent = { + data: { + ordered: { + date: false, + }, + series: [ + { + values: [ + { + xRaw: { + column: 0, + table: { + columns: [ + { + id: '1', + meta: { + type: 'histogram', + indexPatternId: 'indexPatternId', + aggConfigParams: aggConfigs[0].params, + }, + }, + ], + }, + }, + }, + ], + }, + ], + }, + range: [], + }; + }); + + test('should be a function', () => { + expect(typeof onBrushEvent).toBe('function'); + }); + + test('ignores event when data.xAxisField not provided', async () => { + const filter = await onBrushEvent(baseEvent); + expect(filter).toBeUndefined(); + }); + + describe('handles an event when the x-axis field is a date field', () => { + describe('date field is index pattern timefield', () => { + beforeEach(() => { + aggConfigs[0].params.field = { + name: 'time', + type: 'date', + }; + baseEvent.data.ordered = { date: true }; + }); + + afterAll(() => { + baseEvent.range = []; + baseEvent.data.ordered = { date: false }; + }); + + test('by ignoring the event when range spans zero time', async () => { + baseEvent.range = [JAN_01_2014, JAN_01_2014]; + const filter = await onBrushEvent(baseEvent); + expect(filter).toBeUndefined(); + }); + + test('by updating the timefilter', async () => { + baseEvent.range = [JAN_01_2014, JAN_01_2014 + DAY_IN_MS]; + const filter = await onBrushEvent(baseEvent); + expect(filter).toBeDefined(); + + if (filter) { + expect(filter.range.time.gte).toBe(new Date(JAN_01_2014).toISOString()); + // Set to a baseline timezone for comparison. + expect(filter.range.time.lt).toBe(new Date(JAN_01_2014 + DAY_IN_MS).toISOString()); + } + }); + }); + + describe('date field is not index pattern timefield', () => { + beforeEach(() => { + aggConfigs[0].params.field = { + name: 'anotherTimeField', + type: 'date', + }; + baseEvent.data.ordered = { date: true }; + }); + + afterAll(() => { + baseEvent.range = []; + baseEvent.data.ordered = { date: false }; + }); + + test('creates a new range filter', async () => { + const rangeBegin = JAN_01_2014; + const rangeEnd = rangeBegin + DAY_IN_MS; + baseEvent.range = [rangeBegin, rangeEnd]; + const filter = await onBrushEvent(baseEvent); + + expect(filter).toBeDefined(); + + if (filter) { + expect(filter.range.anotherTimeField.gte).toBe(moment(rangeBegin).toISOString()); + expect(filter.range.anotherTimeField.lt).toBe(moment(rangeEnd).toISOString()); + expect(filter.range.anotherTimeField).toHaveProperty( + 'format', + 'strict_date_optional_time' + ); + } + }); + }); + }); + + describe('handles an event when the x-axis field is a number', () => { + beforeAll(() => { + aggConfigs[0].params.field = { + name: 'numberField', + type: 'number', + }; + }); + + afterAll(() => { + baseEvent.range = []; + }); + + test('by ignoring the event when range does not span at least 2 values', async () => { + baseEvent.range = [1]; + const filter = await onBrushEvent(baseEvent); + expect(filter).toBeUndefined(); + }); + + test('by creating a new filter', async () => { + baseEvent.range = [1, 2, 3, 4]; + const filter = await onBrushEvent(baseEvent); + + expect(filter).toBeDefined(); + + if (filter) { + expect(filter.range.numberField.gte).toBe(1); + expect(filter.range.numberField.lt).toBe(4); + expect(filter.range.numberField).not.toHaveProperty('format'); + } + }); + }); +}); diff --git a/src/legacy/core_plugins/data/public/actions/filters/brush_event.ts b/src/legacy/core_plugins/data/public/actions/filters/brush_event.ts new file mode 100644 index 0000000000000..00990d21ccf37 --- /dev/null +++ b/src/legacy/core_plugins/data/public/actions/filters/brush_event.ts @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { get, last } from 'lodash'; +import moment from 'moment'; +import { esFilters, IFieldType, RangeFilterParams } from '../../../../../../plugins/data/public'; +import { deserializeAggConfig } from '../../search/expressions/utils'; +// should be removed after moving into new platform plugins data folder +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { getIndexPatterns } from '../../../../../../plugins/data/public/services'; + +export interface BrushEvent { + data: { + ordered: { + date: boolean; + }; + series: Array>; + }; + range: number[]; +} + +export async function onBrushEvent(event: BrushEvent) { + const isDate = get(event.data, 'ordered.date'); + const xRaw: Record = get(event.data, 'series[0].values[0].xRaw'); + + if (!xRaw) { + return; + } + + const column: Record = xRaw.table.columns[xRaw.column]; + + if (!column || !column.meta) { + return; + } + + const indexPattern = await getIndexPatterns().get(column.meta.indexPatternId); + const aggConfig = deserializeAggConfig({ + ...column.meta, + indexPattern, + }); + const field: IFieldType = aggConfig.params.field; + + if (!field || event.range.length <= 1) { + return; + } + + const min = event.range[0]; + const max = last(event.range); + + if (min === max) { + return; + } + + const range: RangeFilterParams = { + gte: isDate ? moment(min).toISOString() : min, + lt: isDate ? moment(max).toISOString() : max, + }; + + if (isDate) { + range.format = 'strict_date_optional_time'; + } + + return esFilters.buildRangeFilter(field, range, indexPattern); +} diff --git a/src/legacy/core_plugins/data/public/actions/select_range_action.ts b/src/legacy/core_plugins/data/public/actions/select_range_action.ts index 8d0b74be50535..7f1c5d78ab800 100644 --- a/src/legacy/core_plugins/data/public/actions/select_range_action.ts +++ b/src/legacy/core_plugins/data/public/actions/select_range_action.ts @@ -23,16 +23,8 @@ import { createAction, IncompatibleActionError, } from '../../../../../plugins/ui_actions/public'; -// @ts-ignore import { onBrushEvent } from './filters/brush_event'; -import { - Filter, - FilterManager, - TimefilterContract, - esFilters, -} from '../../../../../plugins/data/public'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { getIndexPatterns } from '../../../../../plugins/data/public/services'; +import { FilterManager, TimefilterContract, esFilters } from '../../../../../plugins/data/public'; export const SELECT_RANGE_ACTION = 'SELECT_RANGE_ACTION'; @@ -43,8 +35,7 @@ interface ActionContext { async function isCompatible(context: ActionContext) { try { - const filters: Filter[] = (await onBrushEvent(context.data, getIndexPatterns)) || []; - return filters.length > 0; + return Boolean(await onBrushEvent(context.data)); } catch { return false; } @@ -68,9 +59,13 @@ export function selectRangeAction( throw new IncompatibleActionError(); } - const filters: Filter[] = (await onBrushEvent(data, getIndexPatterns)) || []; + const filter = await onBrushEvent(data); + + if (!filter) { + return; + } - const selectedFilters: Filter[] = esFilters.mapAndFlattenFilters(filters); + const selectedFilters = esFilters.mapAndFlattenFilters([filter]); if (timeFieldName) { const { timeRangeFilter, restOfFilters } = esFilters.extractTimeFilter( diff --git a/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/discover.js b/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/discover.js index bf5049cd976a3..1ac54ad5dabee 100644 --- a/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/discover.js +++ b/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/discover.js @@ -556,8 +556,9 @@ function discoverController( $scope.opts = { // number of records to fetch, then paginate through sampleSize: config.get('discover:sampleSize'), - timefield: - indexPatternsUtils.isDefault($scope.indexPattern) && $scope.indexPattern.timeFieldName, + timefield: indexPatternsUtils.isDefault($scope.indexPattern) + ? $scope.indexPattern.timeFieldName + : undefined, savedSearch: savedSearch, indexPatternList: $route.current.locals.savedObjects.ip.list, }; diff --git a/src/legacy/core_plugins/vis_type_table/public/components/table_vis_options.tsx b/src/legacy/core_plugins/vis_type_table/public/components/table_vis_options.tsx index 5729618b6ae07..8cc0ca2456867 100644 --- a/src/legacy/core_plugins/vis_type_table/public/components/table_vis_options.tsx +++ b/src/legacy/core_plugins/vis_type_table/public/components/table_vis_options.tsx @@ -47,14 +47,14 @@ function TableOptions({ .filter(col => get(col.aggConfig.type.getFormat(col.aggConfig), 'type.id') === 'number') .map(({ name }) => ({ value: name, text: name })), ], - [aggs, stateParams.percentageCol, stateParams.dimensions] + [aggs] ); const isPerPageValid = stateParams.perPage === '' || stateParams.perPage > 0; useEffect(() => { setValidity(isPerPageValid); - }, [isPerPageValid]); + }, [isPerPageValid, setValidity]); useEffect(() => { if ( @@ -64,7 +64,7 @@ function TableOptions({ ) { setValue('percentageCol', percentageColumns[0].value); } - }, [percentageColumns, stateParams.percentageCol]); + }, [percentageColumns, stateParams.percentageCol, setValidity, setValue]); return ( diff --git a/src/plugins/data/public/query/saved_query/saved_query_service.test.ts b/src/plugins/data/public/query/saved_query/saved_query_service.test.ts index a6b8de32a00bd..c983cc4ea8fc5 100644 --- a/src/plugins/data/public/query/saved_query/saved_query_service.test.ts +++ b/src/plugins/data/public/query/saved_query/saved_query_service.test.ts @@ -169,15 +169,27 @@ describe('saved query service', () => { it('should find and return saved queries without search text or pagination parameters', async () => { mockSavedObjectsClient.find.mockReturnValue({ savedObjects: [{ id: 'foo', attributes: savedQueryAttributes }], + total: 5, }); const response = await findSavedQueries(); - expect(response).toEqual([{ id: 'foo', attributes: savedQueryAttributes }]); + expect(response.queries).toEqual([{ id: 'foo', attributes: savedQueryAttributes }]); + }); + + it('should return the total count along with the requested queries', async () => { + mockSavedObjectsClient.find.mockReturnValue({ + savedObjects: [{ id: 'foo', attributes: savedQueryAttributes }], + total: 5, + }); + + const response = await findSavedQueries(); + expect(response.total).toEqual(5); }); it('should find and return saved queries with search text matching the title field', async () => { mockSavedObjectsClient.find.mockReturnValue({ savedObjects: [{ id: 'foo', attributes: savedQueryAttributes }], + total: 5, }); const response = await findSavedQueries('foo'); expect(mockSavedObjectsClient.find).toHaveBeenCalledWith({ @@ -188,7 +200,7 @@ describe('saved query service', () => { sortField: '_score', type: 'query', }); - expect(response).toEqual([{ id: 'foo', attributes: savedQueryAttributes }]); + expect(response.queries).toEqual([{ id: 'foo', attributes: savedQueryAttributes }]); }); it('should find and return parsed filters and timefilters items', async () => { const serializedSavedQueryAttributesWithFilters = { @@ -198,16 +210,20 @@ describe('saved query service', () => { }; mockSavedObjectsClient.find.mockReturnValue({ savedObjects: [{ id: 'foo', attributes: serializedSavedQueryAttributesWithFilters }], + total: 5, }); const response = await findSavedQueries('bar'); - expect(response).toEqual([{ id: 'foo', attributes: savedQueryAttributesWithFilters }]); + expect(response.queries).toEqual([ + { id: 'foo', attributes: savedQueryAttributesWithFilters }, + ]); }); it('should return an array of saved queries', async () => { mockSavedObjectsClient.find.mockReturnValue({ savedObjects: [{ id: 'foo', attributes: savedQueryAttributes }], + total: 5, }); const response = await findSavedQueries(); - expect(response).toEqual( + expect(response.queries).toEqual( expect.objectContaining([ { attributes: { @@ -226,6 +242,7 @@ describe('saved query service', () => { { id: 'foo', attributes: savedQueryAttributes }, { id: 'bar', attributes: savedQueryAttributesBar }, ], + total: 5, }); const response = await findSavedQueries(undefined, 2, 1); expect(mockSavedObjectsClient.find).toHaveBeenCalledWith({ @@ -236,7 +253,7 @@ describe('saved query service', () => { sortField: '_score', type: 'query', }); - expect(response).toEqual( + expect(response.queries).toEqual( expect.objectContaining([ { attributes: { diff --git a/src/plugins/data/public/query/saved_query/saved_query_service.ts b/src/plugins/data/public/query/saved_query/saved_query_service.ts index 80dec1c9373ea..4d3a8f441ce5e 100644 --- a/src/plugins/data/public/query/saved_query/saved_query_service.ts +++ b/src/plugins/data/public/query/saved_query/saved_query_service.ts @@ -95,7 +95,7 @@ export const createSavedQueryService = ( searchText: string = '', perPage: number = 50, activePage: number = 1 - ): Promise => { + ): Promise<{ total: number; queries: SavedQuery[] }> => { const response = await savedObjectsClient.find({ type: 'query', search: searchText, @@ -105,10 +105,13 @@ export const createSavedQueryService = ( page: activePage, }); - return response.savedObjects.map( - (savedObject: { id: string; attributes: SerializedSavedQueryAttributes }) => - parseSavedQueryObject(savedObject) - ); + return { + total: response.total, + queries: response.savedObjects.map( + (savedObject: { id: string; attributes: SerializedSavedQueryAttributes }) => + parseSavedQueryObject(savedObject) + ), + }; }; const getSavedQuery = async (id: string): Promise => { diff --git a/src/plugins/data/public/query/saved_query/types.ts b/src/plugins/data/public/query/saved_query/types.ts index d05eada7b29e6..6ac5e51d5c312 100644 --- a/src/plugins/data/public/query/saved_query/types.ts +++ b/src/plugins/data/public/query/saved_query/types.ts @@ -46,7 +46,7 @@ export interface SavedQueryService { searchText?: string, perPage?: number, activePage?: number - ) => Promise; + ) => Promise<{ total: number; queries: SavedQuery[] }>; getSavedQuery: (id: string) => Promise; deleteSavedQuery: (id: string) => Promise<{}>; getSavedQueryCount: () => Promise; diff --git a/src/plugins/data/public/ui/saved_query_management/saved_query_management_component.tsx b/src/plugins/data/public/ui/saved_query_management/saved_query_management_component.tsx index 2a11531ee336d..9347ef5974261 100644 --- a/src/plugins/data/public/ui/saved_query_management/saved_query_management_component.tsx +++ b/src/plugins/data/public/ui/saved_query_management/saved_query_management_component.tsx @@ -33,7 +33,7 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React, { FunctionComponent, useEffect, useState, Fragment } from 'react'; +import React, { FunctionComponent, useEffect, useState, Fragment, useRef } from 'react'; import { sortBy } from 'lodash'; import { SavedQuery, SavedQueryService } from '../..'; import { SavedQueryListItem } from './saved_query_list_item'; @@ -62,14 +62,25 @@ export const SavedQueryManagementComponent: FunctionComponent = ({ const [savedQueries, setSavedQueries] = useState([] as SavedQuery[]); const [count, setTotalCount] = useState(0); const [activePage, setActivePage] = useState(0); + const cancelPendingListingRequest = useRef<() => void>(() => {}); useEffect(() => { const fetchCountAndSavedQueries = async () => { - const savedQueryCount = await savedQueryService.getSavedQueryCount(); - setTotalCount(savedQueryCount); + cancelPendingListingRequest.current(); + let requestGotCancelled = false; + cancelPendingListingRequest.current = () => { + requestGotCancelled = true; + }; + + const { + total: savedQueryCount, + queries: savedQueryItems, + } = await savedQueryService.findSavedQueries('', perPage, activePage + 1); + + if (requestGotCancelled) return; - const savedQueryItems = await savedQueryService.findSavedQueries('', perPage, activePage + 1); const sortedSavedQueryItems = sortBy(savedQueryItems, 'attributes.title'); + setTotalCount(savedQueryCount); setSavedQueries(sortedSavedQueryItems); }; if (isOpen) { @@ -103,6 +114,7 @@ export const SavedQueryManagementComponent: FunctionComponent = ({ ); const onDeleteSavedQuery = async (savedQuery: SavedQuery) => { + cancelPendingListingRequest.current(); setSavedQueries( savedQueries.filter(currentSavedQuery => currentSavedQuery.id !== savedQuery.id) ); diff --git a/test/functional/apps/dashboard/dashboard_filtering.js b/test/functional/apps/dashboard/dashboard_filtering.js index 1cb9f1490d442..ec8a48ca74911 100644 --- a/test/functional/apps/dashboard/dashboard_filtering.js +++ b/test/functional/apps/dashboard/dashboard_filtering.js @@ -63,6 +63,10 @@ export default function({ getService, getPageObjects }) { await filterBar.addFilter('bytes', 'is', '12345678'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.dashboard.waitForRenderComplete(); + // first round of requests sometimes times out, refresh all visualizations to fetch again + await queryBar.clickQuerySubmitButton(); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.dashboard.waitForRenderComplete(); }); it('filters on pie charts', async () => { diff --git a/test/functional/services/saved_query_management_component.ts b/test/functional/services/saved_query_management_component.ts index 9f0a8ded649b2..b94558c209e6a 100644 --- a/test/functional/services/saved_query_management_component.ts +++ b/test/functional/services/saved_query_management_component.ts @@ -24,6 +24,7 @@ export function SavedQueryManagementComponentProvider({ getService }: FtrProvide const testSubjects = getService('testSubjects'); const queryBar = getService('queryBar'); const retry = getService('retry'); + const config = getService('config'); class SavedQueryManagementComponent { public async getCurrentlyLoadedQueryID() { @@ -177,7 +178,9 @@ export function SavedQueryManagementComponentProvider({ getService }: FtrProvide await retry.try(async () => { await testSubjects.click('saved-query-management-save-button'); - await testSubjects.existOrFail('saveQueryForm'); + await testSubjects.existOrFail('saveQueryForm', { + timeout: config.get('timeouts.waitForExists'), + }); }); } diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/PlatinumLicensePrompt.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/PlatinumLicensePrompt.tsx index c5771995daa24..9213349a1492b 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/PlatinumLicensePrompt.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/PlatinumLicensePrompt.tsx @@ -4,15 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import React from 'react'; import { - EuiEmptyPrompt, EuiButton, - EuiPanel, + EuiEmptyPrompt, EuiFlexGroup, - EuiFlexItem + EuiFlexItem, + EuiPanel } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { invalidLicenseMessage } from '../../../../../../../plugins/apm/common/service_map'; import { useKibanaUrl } from '../../../hooks/useKibanaUrl'; export function PlatinumLicensePrompt() { @@ -43,14 +44,7 @@ export function PlatinumLicensePrompt() { )} ]} - body={ -

- {i18n.translate('xpack.apm.serviceMap.licensePromptBody', { - defaultMessage: - "In order to access Service Maps, you must be subscribed to an Elastic Platinum license. With it, you'll have the ability to visualize your entire application stack along with your APM data." - })} -

- } + body={

{invalidLicenseMessage}

} title={

{i18n.translate('xpack.apm.serviceMap.licensePromptTitle', { diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx index 5fea4be9ca0da..b14ecaa803f6d 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/index.tsx @@ -17,12 +17,14 @@ import React, { useState } from 'react'; import { toMountPoint } from '../../../../../../../../src/plugins/kibana_react/public'; +import { isValidPlatinumLicense } from '../../../../../../../plugins/apm/common/service_map'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { ServiceMapAPIResponse } from '../../../../../../../plugins/apm/server/lib/service_map/get_service_map'; import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; import { useCallApmApi } from '../../../hooks/useCallApmApi'; import { useDeepObjectIdentity } from '../../../hooks/useDeepObjectIdentity'; import { useLicense } from '../../../hooks/useLicense'; +import { useLoadingIndicator } from '../../../hooks/useLoadingIndicator'; import { useLocation } from '../../../hooks/useLocation'; import { useUrlParams } from '../../../hooks/useUrlParams'; import { Controls } from './Controls'; @@ -31,7 +33,6 @@ import { getCytoscapeElements } from './get_cytoscape_elements'; import { PlatinumLicensePrompt } from './PlatinumLicensePrompt'; import { Popover } from './Popover'; import { useRefHeight } from './useRefHeight'; -import { useLoadingIndicator } from '../../../hooks/useLoadingIndicator'; interface ServiceMapProps { serviceName?: string; @@ -195,13 +196,13 @@ export function ServiceMap({ serviceName }: ServiceMapProps) { // eslint-disable-next-line react-hooks/exhaustive-deps }, [elements]); - const isValidPlatinumLicense = - license?.isActive && - (license?.type === 'platinum' || license?.type === 'trial'); - const [wrapperRef, height] = useRefHeight(); - return isValidPlatinumLicense ? ( + if (!license) { + return null; + } + + return isValidPlatinumLicense(license) ? (
dispatch(startDataLoad(layerId, dataId, requestToken, meta)), @@ -87,6 +88,13 @@ function getLayerLoadingCallbacks(dispatch, layerId) { updateSourceData: newData => { dispatch(updateSourceDataRequest(layerId, newData)); }, + isRequestStillActive: (dataId, requestToken) => { + const dataRequest = getDataRequestDescriptor(getState(), layerId, dataId); + if (!dataRequest) { + return false; + } + return dataRequest.dataRequestToken === requestToken; + }, registerCancelCallback: (requestToken, callback) => dispatch(registerCancelCallback(requestToken, callback)), }; @@ -98,11 +106,11 @@ function getLayerById(layerId, state) { }); } -async function syncDataForAllLayers(getState, dispatch, dataFilters) { +async function syncDataForAllLayers(dispatch, getState, dataFilters) { const state = getState(); const layerList = getLayerList(state); const syncs = layerList.map(layer => { - const loadingFunctions = getLayerLoadingCallbacks(dispatch, layer.getId()); + const loadingFunctions = getLayerLoadingCallbacks(dispatch, getState, layer.getId()); return layer.syncData({ ...loadingFunctions, dataFilters }); }); await Promise.all(syncs); @@ -412,7 +420,7 @@ export function mapExtentChanged(newMapConstants) { }, }); const newDataFilters = { ...dataFilters, ...newMapConstants }; - await syncDataForAllLayers(getState, dispatch, newDataFilters); + await syncDataForAllLayers(dispatch, getState, newDataFilters); }; } @@ -653,7 +661,7 @@ export function syncDataForLayer(layerId) { const targetLayer = getLayerById(layerId, getState()); if (targetLayer) { const dataFilters = getDataFilters(getState()); - const loadingFunctions = getLayerLoadingCallbacks(dispatch, layerId); + const loadingFunctions = getLayerLoadingCallbacks(dispatch, getState, layerId); await targetLayer.syncData({ ...loadingFunctions, dataFilters, @@ -773,7 +781,7 @@ export function setQuery({ query, timeFilters, filters = [], refresh = false }) }); const dataFilters = getDataFilters(getState()); - await syncDataForAllLayers(getState, dispatch, dataFilters); + await syncDataForAllLayers(dispatch, getState, dataFilters); }; } @@ -792,7 +800,7 @@ export function triggerRefreshTimer() { }); const dataFilters = getDataFilters(getState()); - await syncDataForAllLayers(getState, dispatch, dataFilters); + await syncDataForAllLayers(dispatch, getState, dataFilters); }; } diff --git a/x-pack/legacy/plugins/maps/public/components/metric_editor.js b/x-pack/legacy/plugins/maps/public/components/metric_editor.js index e60c2ac0dd7ab..530f402592b2b 100644 --- a/x-pack/legacy/plugins/maps/public/components/metric_editor.js +++ b/x-pack/legacy/plugins/maps/public/components/metric_editor.js @@ -12,17 +12,16 @@ import { EuiFieldText, EuiFormRow } from '@elastic/eui'; import { MetricSelect, METRIC_AGGREGATION_VALUES } from './metric_select'; import { SingleFieldSelect } from './single_field_select'; -import { METRIC_TYPE } from '../../common/constants'; +import { AGG_TYPE } from '../../common/constants'; +import { getTermsFields } from '../index_pattern_util'; function filterFieldsForAgg(fields, aggType) { if (!fields) { return []; } - if (aggType === METRIC_TYPE.UNIQUE_COUNT) { - return fields.filter(field => { - return field.aggregatable; - }); + if (aggType === AGG_TYPE.UNIQUE_COUNT || aggType === AGG_TYPE.TERMS) { + return getTermsFields(fields); } return fields.filter(field => { @@ -38,7 +37,7 @@ export function MetricEditor({ fields, metricsFilter, metric, onChange, removeBu }; // unset field when new agg type does not support currently selected field. - if (metric.field && metricAggregationType !== METRIC_TYPE.COUNT) { + if (metric.field && metricAggregationType !== AGG_TYPE.COUNT) { const fieldsForNewAggType = filterFieldsForAgg(fields, metricAggregationType); const found = fieldsForNewAggType.find(field => { return field.name === metric.field; @@ -64,7 +63,7 @@ export function MetricEditor({ fields, metricsFilter, metric, onChange, removeBu }; let fieldSelect; - if (metric.type && metric.type !== METRIC_TYPE.COUNT) { + if (metric.type && metric.type !== AGG_TYPE.COUNT) { fieldSelect = ( { - if (type === METRIC_TYPE.COUNT) { + if (type === AGG_TYPE.COUNT) { return true; } @@ -70,7 +70,7 @@ export class MetricsExpression extends Component { }) .map(({ type, field }) => { // do not use metric label so field and aggregation are not obscured. - if (type === METRIC_TYPE.COUNT) { + if (type === AGG_TYPE.COUNT) { return 'count'; } @@ -130,5 +130,5 @@ MetricsExpression.propTypes = { }; MetricsExpression.defaultProps = { - metrics: [{ type: METRIC_TYPE.COUNT }], + metrics: [{ type: AGG_TYPE.COUNT }], }; diff --git a/x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js b/x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js index e0e1556ecde06..e4e3776c8e92c 100644 --- a/x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js +++ b/x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js @@ -4,6 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ +jest.mock('../../../../components/metrics_editor', () => ({ + MetricsEditor: () => { + return
mockMetricsEditor
; + }, +})); + import React from 'react'; import { shallow } from 'enzyme'; import { MetricsExpression } from './metrics_expression'; diff --git a/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js b/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js index 1e44c7225a564..fdc8ad2176d08 100644 --- a/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js +++ b/x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js @@ -23,6 +23,7 @@ import sprites1 from '@elastic/maki/dist/sprite@1.png'; import sprites2 from '@elastic/maki/dist/sprite@2.png'; import { DrawControl } from './draw_control'; import { TooltipControl } from './tooltip_control'; +import { clampToLatBounds, clampToLonBounds } from '../../../elasticsearch_geo_utils'; mapboxgl.workerUrl = mbWorkerUrl; mapboxgl.setRTLTextPlugin(mbRtlPlugin); @@ -234,12 +235,12 @@ export class MBMapContainer extends React.Component { //clamping ot -89/89 latitudes since Mapboxgl does not seem to handle bounds that contain the poles (logs errors to the console when using -90/90) const lnLatBounds = new mapboxgl.LngLatBounds( new mapboxgl.LngLat( - clamp(goto.bounds.min_lon, -180, 180), - clamp(goto.bounds.min_lat, -89, 89) + clampToLonBounds(goto.bounds.min_lon), + clampToLatBounds(goto.bounds.min_lat) ), new mapboxgl.LngLat( - clamp(goto.bounds.max_lon, -180, 180), - clamp(goto.bounds.max_lat, -89, 89) + clampToLonBounds(goto.bounds.max_lon), + clampToLatBounds(goto.bounds.max_lat) ) ); //maxZoom ensure we're not zooming in too far on single points or small shapes @@ -306,9 +307,3 @@ export class MBMapContainer extends React.Component { ); } } - -function clamp(val, min, max) { - if (val > max) val = max; - else if (val < min) val = min; - return val; -} diff --git a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js index ec0ae4161b3f2..9b33d3036785c 100644 --- a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js +++ b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js @@ -433,3 +433,21 @@ export function convertMapExtentToPolygon({ maxLat, maxLon, minLat, minLon }) { return formatEnvelopeAsPolygon({ maxLat, maxLon, minLat, minLon }); } + +export function clampToLatBounds(lat) { + return clamp(lat, -89, 89); +} + +export function clampToLonBounds(lon) { + return clamp(lon, -180, 180); +} + +export function clamp(val, min, max) { + if (val > max) { + return max; + } else if (val < min) { + return min; + } else { + return val; + } +} diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js index 65109cb99809f..28c199b64d3ef 100644 --- a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js +++ b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js @@ -5,9 +5,10 @@ */ import { AbstractField } from './field'; -import { COUNT_AGG_TYPE } from '../../../common/constants'; +import { AGG_TYPE } from '../../../common/constants'; import { isMetricCountable } from '../util/is_metric_countable'; import { ESAggMetricTooltipProperty } from '../tooltips/es_aggmetric_tooltip_property'; +import { getField, addFieldToDSL } from '../util/es_agg_utils'; export class ESAggMetricField extends AbstractField { static type = 'ES_AGG'; @@ -34,12 +35,11 @@ export class ESAggMetricField extends AbstractField { } isValid() { - return this.getAggType() === COUNT_AGG_TYPE ? true : !!this._esDocField; + return this.getAggType() === AGG_TYPE.COUNT ? true : !!this._esDocField; } async getDataType() { - // aggregations only provide numerical data - return 'number'; + return this.getAggType() === AGG_TYPE.TERMS ? 'string' : 'number'; } getESDocFieldName() { @@ -47,9 +47,9 @@ export class ESAggMetricField extends AbstractField { } getRequestDescription() { - return this.getAggType() !== COUNT_AGG_TYPE + return this.getAggType() !== AGG_TYPE.COUNT ? `${this.getAggType()} ${this.getESDocFieldName()}` - : COUNT_AGG_TYPE; + : AGG_TYPE.COUNT; } async createTooltipProperty(value) { @@ -63,18 +63,13 @@ export class ESAggMetricField extends AbstractField { ); } - makeMetricAggConfig() { - const metricAggConfig = { - id: this.getName(), - enabled: true, - type: this.getAggType(), - schema: 'metric', - params: {}, + getValueAggDsl(indexPattern) { + const field = getField(indexPattern, this.getESDocFieldName()); + const aggType = this.getAggType(); + const aggBody = aggType === AGG_TYPE.TERMS ? { size: 1, shard_size: 1 } : {}; + return { + [aggType]: addFieldToDSL(aggBody, field), }; - if (this.getAggType() !== COUNT_AGG_TYPE) { - metricAggConfig.params = { field: this.getESDocFieldName() }; - } - return metricAggConfig; } supportsFieldMeta() { @@ -85,4 +80,8 @@ export class ESAggMetricField extends AbstractField { async getOrdinalFieldMetaRequest(config) { return this._esDocField.getOrdinalFieldMetaRequest(config); } + + async getCategoricalFieldMetaRequest() { + return this._esDocField.getCategoricalFieldMetaRequest(); + } } diff --git a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js index 2f18987513d92..aeeffd63607ee 100644 --- a/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js @@ -5,24 +5,24 @@ */ import { ESAggMetricField } from './es_agg_field'; -import { METRIC_TYPE } from '../../../common/constants'; +import { AGG_TYPE } from '../../../common/constants'; describe('supportsFieldMeta', () => { test('Non-counting aggregations should support field meta', () => { - const avgMetric = new ESAggMetricField({ aggType: METRIC_TYPE.AVG }); + const avgMetric = new ESAggMetricField({ aggType: AGG_TYPE.AVG }); expect(avgMetric.supportsFieldMeta()).toBe(true); - const maxMetric = new ESAggMetricField({ aggType: METRIC_TYPE.MAX }); + const maxMetric = new ESAggMetricField({ aggType: AGG_TYPE.MAX }); expect(maxMetric.supportsFieldMeta()).toBe(true); - const minMetric = new ESAggMetricField({ aggType: METRIC_TYPE.MIN }); + const minMetric = new ESAggMetricField({ aggType: AGG_TYPE.MIN }); expect(minMetric.supportsFieldMeta()).toBe(true); }); test('Counting aggregations should not support field meta', () => { - const countMetric = new ESAggMetricField({ aggType: METRIC_TYPE.COUNT }); + const countMetric = new ESAggMetricField({ aggType: AGG_TYPE.COUNT }); expect(countMetric.supportsFieldMeta()).toBe(false); - const sumMetric = new ESAggMetricField({ aggType: METRIC_TYPE.SUM }); + const sumMetric = new ESAggMetricField({ aggType: AGG_TYPE.SUM }); expect(sumMetric.supportsFieldMeta()).toBe(false); - const uniqueCountMetric = new ESAggMetricField({ aggType: METRIC_TYPE.UNIQUE_COUNT }); + const uniqueCountMetric = new ESAggMetricField({ aggType: AGG_TYPE.UNIQUE_COUNT }); expect(uniqueCountMetric.supportsFieldMeta()).toBe(false); }); }); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js index 967a3c41aec26..bee35216f59da 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js @@ -8,8 +8,7 @@ import { AbstractESSource } from './es_source'; import { ESAggMetricField } from '../fields/es_agg_field'; import { ESDocField } from '../fields/es_doc_field'; import { - METRIC_TYPE, - COUNT_AGG_TYPE, + AGG_TYPE, COUNT_PROP_LABEL, COUNT_PROP_NAME, FIELD_ORIGIN, @@ -18,23 +17,6 @@ import { export const AGG_DELIMITER = '_of_'; export class AbstractESAggSource extends AbstractESSource { - static METRIC_SCHEMA_CONFIG = { - group: 'metrics', - name: 'metric', - title: 'Value', - min: 1, - max: Infinity, - aggFilter: [ - METRIC_TYPE.AVG, - METRIC_TYPE.COUNT, - METRIC_TYPE.MAX, - METRIC_TYPE.MIN, - METRIC_TYPE.SUM, - METRIC_TYPE.UNIQUE_COUNT, - ], - defaults: [{ schema: 'metric', type: METRIC_TYPE.COUNT }], - }; - constructor(descriptor, inspectorAdapters) { super(descriptor, inspectorAdapters); this._metricFields = this._descriptor.metrics @@ -81,7 +63,7 @@ export class AbstractESAggSource extends AbstractESSource { if (metrics.length === 0) { metrics.push( new ESAggMetricField({ - aggType: COUNT_AGG_TYPE, + aggType: AGG_TYPE.COUNT, source: this, origin: this.getOriginForField(), }) @@ -91,15 +73,23 @@ export class AbstractESAggSource extends AbstractESSource { } formatMetricKey(aggType, fieldName) { - return aggType !== COUNT_AGG_TYPE ? `${aggType}${AGG_DELIMITER}${fieldName}` : COUNT_PROP_NAME; + return aggType !== AGG_TYPE.COUNT ? `${aggType}${AGG_DELIMITER}${fieldName}` : COUNT_PROP_NAME; } formatMetricLabel(aggType, fieldName) { - return aggType !== COUNT_AGG_TYPE ? `${aggType} of ${fieldName}` : COUNT_PROP_LABEL; + return aggType !== AGG_TYPE.COUNT ? `${aggType} of ${fieldName}` : COUNT_PROP_LABEL; } - createMetricAggConfigs() { - return this.getMetricFields().map(esAggMetric => esAggMetric.makeMetricAggConfig()); + getValueAggsDsl(indexPattern) { + const valueAggsDsl = {}; + this.getMetricFields() + .filter(esAggMetric => { + return esAggMetric.getAggType() !== AGG_TYPE.COUNT; + }) + .forEach(esAggMetric => { + valueAggsDsl[esAggMetric.getName()] = esAggMetric.getValueAggDsl(indexPattern); + }); + return valueAggsDsl; } async getNumberFields() { diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js index 4e15d1c927c36..bb9bf1b508f94 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js @@ -4,68 +4,63 @@ * you may not use this file except in compliance with the Elastic License. */ +import _ from 'lodash'; import { RENDER_AS } from './render_as'; import { getTileBoundingBox } from './geo_tile_utils'; -import { EMPTY_FEATURE_COLLECTION } from '../../../../common/constants'; +import { extractPropertiesFromBucket } from '../../util/es_agg_utils'; +import { clamp } from '../../../elasticsearch_geo_utils'; -export function convertToGeoJson({ table, renderAs }) { - if (!table || !table.rows) { - return EMPTY_FEATURE_COLLECTION; - } +const GRID_BUCKET_KEYS_TO_IGNORE = ['key', 'gridCentroid']; - const geoGridColumn = table.columns.find( - column => column.aggConfig.type.dslName === 'geotile_grid' +export function convertCompositeRespToGeoJson(esResponse, renderAs) { + return convertToGeoJson( + esResponse, + renderAs, + esResponse => { + return _.get(esResponse, 'aggregations.compositeSplit.buckets', []); + }, + gridBucket => { + return gridBucket.key.gridSplit; + } ); - if (!geoGridColumn) { - return EMPTY_FEATURE_COLLECTION; - } +} - const metricColumns = table.columns.filter(column => { - return ( - column.aggConfig.type.type === 'metrics' && column.aggConfig.type.dslName !== 'geo_centroid' - ); - }); - const geocentroidColumn = table.columns.find( - column => column.aggConfig.type.dslName === 'geo_centroid' +export function convertRegularRespToGeoJson(esResponse, renderAs) { + return convertToGeoJson( + esResponse, + renderAs, + esResponse => { + return _.get(esResponse, 'aggregations.gridSplit.buckets', []); + }, + gridBucket => { + return gridBucket.key; + } ); - if (!geocentroidColumn) { - return EMPTY_FEATURE_COLLECTION; - } +} +function convertToGeoJson(esResponse, renderAs, pluckGridBuckets, pluckGridKey) { const features = []; - table.rows.forEach(row => { - const gridKey = row[geoGridColumn.id]; - if (!gridKey) { - return; - } - - const properties = {}; - metricColumns.forEach(metricColumn => { - properties[metricColumn.aggConfig.id] = row[metricColumn.id]; - }); + const gridBuckets = pluckGridBuckets(esResponse); + for (let i = 0; i < gridBuckets.length; i++) { + const gridBucket = gridBuckets[i]; + const gridKey = pluckGridKey(gridBucket); features.push({ type: 'Feature', geometry: rowToGeometry({ - row, gridKey, - geocentroidColumn, + gridCentroid: gridBucket.gridCentroid, renderAs, }), id: gridKey, - properties, + properties: extractPropertiesFromBucket(gridBucket, GRID_BUCKET_KEYS_TO_IGNORE), }); - }); + } - return { - featureCollection: { - type: 'FeatureCollection', - features: features, - }, - }; + return features; } -function rowToGeometry({ row, gridKey, geocentroidColumn, renderAs }) { +function rowToGeometry({ gridKey, gridCentroid, renderAs }) { const { top, bottom, right, left } = getTileBoundingBox(gridKey); if (renderAs === RENDER_AS.GRID) { @@ -83,10 +78,10 @@ function rowToGeometry({ row, gridKey, geocentroidColumn, renderAs }) { }; } - // see https://github.com/elastic/elasticsearch/issues/24694 for why clampGrid is used + // see https://github.com/elastic/elasticsearch/issues/24694 for why clamp is used const pointCoordinates = [ - clampGrid(row[geocentroidColumn.id].lon, left, right), - clampGrid(row[geocentroidColumn.id].lat, bottom, top), + clamp(gridCentroid.location.lon, left, right), + clamp(gridCentroid.location.lat, bottom, top), ]; return { @@ -94,9 +89,3 @@ function rowToGeometry({ row, gridKey, geocentroidColumn, renderAs }) { coordinates: pointCoordinates, }; } - -function clampGrid(val, min, max) { - if (val > max) val = max; - else if (val < min) val = min; - return val; -} diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts new file mode 100644 index 0000000000000..ba79464a01a9b --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +jest.mock('../../../kibana_services', () => {}); + +// @ts-ignore +import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson'; +// @ts-ignore +import { RENDER_AS } from './render_as'; + +describe('convertCompositeRespToGeoJson', () => { + const esResponse = { + aggregations: { + compositeSplit: { + after_key: { + gridSplit: '10/327/460', + }, + buckets: [ + { + key: { gridSplit: '4/4/6' }, + doc_count: 65, + avg_of_bytes: { value: 5359.2307692307695 }, + 'terms_of_machine.os.keyword': { + buckets: [ + { + key: 'win xp', + doc_count: 16, + }, + ], + }, + gridCentroid: { + location: { lat: 36.62813963153614, lon: -81.94552666092149 }, + count: 65, + }, + }, + ], + }, + }, + }; + + it('Should convert elasticsearch aggregation response into feature collection of points', () => { + const features = convertCompositeRespToGeoJson(esResponse, RENDER_AS.POINT); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ + geometry: { + coordinates: [-81.94552666092149, 36.62813963153614], + type: 'Point', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); + }); + + it('Should convert elasticsearch aggregation response into feature collection of Polygons', () => { + const features = convertCompositeRespToGeoJson(esResponse, RENDER_AS.GRID); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ + geometry: { + coordinates: [ + [ + [-67.5, 40.9799], + [-90, 40.9799], + [-90, 21.94305], + [-67.5, 21.94305], + [-67.5, 40.9799], + ], + ], + type: 'Polygon', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); + }); +}); + +describe('convertRegularRespToGeoJson', () => { + const esResponse = { + aggregations: { + gridSplit: { + buckets: [ + { + key: '4/4/6', + doc_count: 65, + avg_of_bytes: { value: 5359.2307692307695 }, + 'terms_of_machine.os.keyword': { + buckets: [ + { + key: 'win xp', + doc_count: 16, + }, + ], + }, + gridCentroid: { + location: { lat: 36.62813963153614, lon: -81.94552666092149 }, + count: 65, + }, + }, + ], + }, + }, + }; + + it('Should convert elasticsearch aggregation response into feature collection of points', () => { + const features = convertRegularRespToGeoJson(esResponse, RENDER_AS.POINT); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ + geometry: { + coordinates: [-81.94552666092149, 36.62813963153614], + type: 'Point', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); + }); + + it('Should convert elasticsearch aggregation response into feature collection of Polygons', () => { + const features = convertRegularRespToGeoJson(esResponse, RENDER_AS.GRID); + expect(features.length).toBe(1); + expect(features[0]).toEqual({ + geometry: { + coordinates: [ + [ + [-67.5, 40.9799], + [-90, 40.9799], + [-90, 21.94305], + [-67.5, 21.94305], + [-67.5, 40.9799], + ], + ], + type: 'Polygon', + }, + id: '4/4/6', + properties: { + avg_of_bytes: 5359.2307692307695, + doc_count: 65, + 'terms_of_machine.os.keyword': 'win xp', + }, + type: 'Feature', + }); + }); +}); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js index 0912e5a9f1283..a0ddf584bcebc 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js @@ -10,9 +10,7 @@ import uuid from 'uuid/v4'; import { VECTOR_SHAPE_TYPES } from '../vector_feature_types'; import { HeatmapLayer } from '../../heatmap_layer'; import { VectorLayer } from '../../vector_layer'; -import { AggConfigs, Schemas } from 'ui/agg_types'; -import { tabifyAggResponse } from '../../../../../../../../src/legacy/core_plugins/data/public'; -import { convertToGeoJson } from './convert_to_geojson'; +import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson'; import { VectorStyle } from '../../styles/vector/vector_style'; import { getDefaultDynamicProperties, @@ -24,6 +22,8 @@ import { CreateSourceEditor } from './create_source_editor'; import { UpdateSourceEditor } from './update_source_editor'; import { GRID_RESOLUTION } from '../../grid_resolution'; import { + AGG_TYPE, + DEFAULT_MAX_BUCKETS_LIMIT, SOURCE_DATA_ID_ORIGIN, ES_GEO_GRID, COUNT_PROP_NAME, @@ -34,21 +34,10 @@ import { getDataSourceLabel } from '../../../../common/i18n_getters'; import { AbstractESAggSource } from '../es_agg_source'; import { DynamicStyleProperty } from '../../styles/vector/properties/dynamic_style_property'; import { StaticStyleProperty } from '../../styles/vector/properties/static_style_property'; +import { DataRequestAbortError } from '../../util/data_request'; const MAX_GEOTILE_LEVEL = 29; -const aggSchemas = new Schemas([ - AbstractESAggSource.METRIC_SCHEMA_CONFIG, - { - group: 'buckets', - name: 'segment', - title: 'Geo Grid', - aggFilter: 'geotile_grid', - min: 1, - max: 1, - }, -]); - export class ESGeoGridSource extends AbstractESAggSource { static type = ES_GEO_GRID; static title = i18n.translate('xpack.maps.source.esGridTitle', { @@ -175,15 +164,120 @@ export class ESGeoGridSource extends AbstractESAggSource { ); } - async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { - const indexPattern = await this.getIndexPattern(); - const searchSource = await this._makeSearchSource(searchFilters, 0); - const aggConfigs = new AggConfigs( - indexPattern, - this._makeAggConfigs(searchFilters.geogridPrecision), - aggSchemas.all - ); - searchSource.setField('aggs', aggConfigs.toDsl()); + async _compositeAggRequest({ + searchSource, + indexPattern, + precision, + layerName, + registerCancelCallback, + bucketsPerGrid, + isRequestStillActive, + }) { + const gridsPerRequest = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid); + const aggs = { + compositeSplit: { + composite: { + size: gridsPerRequest, + sources: [ + { + gridSplit: { + geotile_grid: { + field: this._descriptor.geoField, + precision, + }, + }, + }, + ], + }, + aggs: { + gridCentroid: { + geo_centroid: { + field: this._descriptor.geoField, + }, + }, + ...this.getValueAggsDsl(indexPattern), + }, + }, + }; + + const features = []; + let requestCount = 0; + let afterKey = null; + while (true) { + if (!isRequestStillActive()) { + // Stop paging through results if request is obsolete + throw new DataRequestAbortError(); + } + + requestCount++; + + // circuit breaker to ensure reasonable number of requests + if (requestCount > 5) { + throw new Error( + i18n.translate('xpack.maps.source.esGrid.compositePaginationErrorMessage', { + defaultMessage: `{layerName} is causing too many requests. Reduce "Grid resolution" and/or reduce the number of top term "Metrics".`, + values: { layerName }, + }) + ); + } + + if (afterKey) { + aggs.compositeSplit.composite.after = afterKey; + } + searchSource.setField('aggs', aggs); + const requestId = afterKey ? `${this.getId()} afterKey ${afterKey.geoSplit}` : this.getId(); + const esResponse = await this._runEsQuery({ + requestId, + requestName: `${layerName} (${requestCount})`, + searchSource, + registerCancelCallback, + requestDescription: i18n.translate( + 'xpack.maps.source.esGrid.compositeInspectorDescription', + { + defaultMessage: 'Elasticsearch geo grid aggregation request: {requestId}', + values: { requestId }, + } + ), + }); + + features.push(...convertCompositeRespToGeoJson(esResponse, this._descriptor.requestType)); + + afterKey = esResponse.aggregations.compositeSplit.after_key; + if (esResponse.aggregations.compositeSplit.buckets.length < gridsPerRequest) { + // Finished because request did not get full resultset back + break; + } + } + + return features; + } + + // Do not use composite aggregation when there are no terms sub-aggregations + // see https://github.com/elastic/kibana/pull/57875#issuecomment-590515482 for explanation on using separate code paths + async _nonCompositeAggRequest({ + searchSource, + indexPattern, + precision, + layerName, + registerCancelCallback, + }) { + searchSource.setField('aggs', { + gridSplit: { + geotile_grid: { + field: this._descriptor.geoField, + precision, + }, + aggs: { + gridCentroid: { + geo_centroid: { + field: this._descriptor.geoField, + }, + }, + ...this.getValueAggsDsl(indexPattern), + }, + }, + }); + const esResponse = await this._runEsQuery({ requestId: this.getId(), requestName: layerName, @@ -194,14 +288,45 @@ export class ESGeoGridSource extends AbstractESAggSource { }), }); - const tabifiedResp = tabifyAggResponse(aggConfigs, esResponse); - const { featureCollection } = convertToGeoJson({ - table: tabifiedResp, - renderAs: this._descriptor.requestType, + return convertRegularRespToGeoJson(esResponse, this._descriptor.requestType); + } + + async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback, isRequestStillActive) { + const indexPattern = await this.getIndexPattern(); + const searchSource = await this._makeSearchSource(searchFilters, 0); + + let bucketsPerGrid = 1; + this.getMetricFields().forEach(metricField => { + if (metricField.getAggType() === AGG_TYPE.TERMS) { + // each terms aggregation increases the overall number of buckets per grid + bucketsPerGrid++; + } }); + const features = + bucketsPerGrid === 1 + ? await this._nonCompositeAggRequest({ + searchSource, + indexPattern, + precision: searchFilters.geogridPrecision, + layerName, + registerCancelCallback, + }) + : await this._compositeAggRequest({ + searchSource, + indexPattern, + precision: searchFilters.geogridPrecision, + layerName, + registerCancelCallback, + bucketsPerGrid, + isRequestStillActive, + }); + return { - data: featureCollection, + data: { + type: 'FeatureCollection', + features: features, + }, meta: { areResultsTrimmed: false, }, @@ -212,24 +337,6 @@ export class ESGeoGridSource extends AbstractESAggSource { return true; } - _makeAggConfigs(precision) { - const metricAggConfigs = this.createMetricAggConfigs(); - return [ - ...metricAggConfigs, - { - id: 'grid', - enabled: true, - type: 'geotile_grid', - schema: 'segment', - params: { - field: this._descriptor.geoField, - useGeocentroid: true, - precision: precision, - }, - }, - ]; - } - _createHeatmapLayerDescriptor(options) { return HeatmapLayer.createDescriptor({ sourceDescriptor: this._descriptor, diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js index da0bc1685f223..251e33b9579cb 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js @@ -6,6 +6,7 @@ import _ from 'lodash'; import { DECIMAL_DEGREES_PRECISION } from '../../../../common/constants'; +import { clampToLatBounds } from '../../../elasticsearch_geo_utils'; const ZOOM_TILE_KEY_INDEX = 0; const X_TILE_KEY_INDEX = 1; @@ -87,7 +88,7 @@ function sec(value) { } function latitudeToTile(lat, tileCount) { - const radians = (lat * Math.PI) / 180; + const radians = (clampToLatBounds(lat) * Math.PI) / 180; const y = ((1 - Math.log(Math.tan(radians) + sec(radians)) / Math.PI) / 2) * tileCount; return Math.floor(y); } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.test.js index ae2623e168766..88a6ce048a178 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.test.js @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +jest.mock('../../../kibana_services', () => {}); + import { parseTileKey, getTileBoundingBox, expandToTileBoundaries } from './geo_tile_utils'; it('Should parse tile key', () => { diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js index 2057949c30c88..96a7f50cdf523 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js @@ -5,9 +5,11 @@ */ import _ from 'lodash'; +import { extractPropertiesFromBucket } from '../../util/es_agg_utils'; const LAT_INDEX = 0; const LON_INDEX = 1; +const PEW_PEW_BUCKET_KEYS_TO_IGNORE = ['key', 'sourceCentroid']; function parsePointFromKey(key) { const split = key.split(','); @@ -25,25 +27,16 @@ export function convertToLines(esResponse) { const dest = parsePointFromKey(destBucket.key); const sourceBuckets = _.get(destBucket, 'sourceGrid.buckets', []); for (let j = 0; j < sourceBuckets.length; j++) { - const { key, sourceCentroid, ...rest } = sourceBuckets[j]; - - // flatten metrics - Object.keys(rest).forEach(key => { - if (_.has(rest[key], 'value')) { - rest[key] = rest[key].value; - } - }); - + const sourceBucket = sourceBuckets[j]; + const sourceCentroid = sourceBucket.sourceCentroid; lineFeatures.push({ type: 'Feature', geometry: { type: 'LineString', coordinates: [[sourceCentroid.location.lon, sourceCentroid.location.lat], dest], }, - id: `${dest.join()},${key}`, - properties: { - ...rest, - }, + id: `${dest.join()},${sourceBucket.key}`, + properties: extractPropertiesFromBucket(sourceBucket, PEW_PEW_BUCKET_KEYS_TO_IGNORE), }); } } diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts new file mode 100644 index 0000000000000..5fbd5a3ad20c0 --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// @ts-ignore +import { convertToLines } from './convert_to_lines'; + +const esResponse = { + aggregations: { + destSplit: { + buckets: [ + { + key: '43.68389896117151, 10.39269994944334', + doc_count: 2, + sourceGrid: { + buckets: [ + { + key: '4/9/3', + doc_count: 1, + terms_of_Carrier: { + buckets: [ + { + key: 'ES-Air', + doc_count: 1, + }, + ], + }, + sourceCentroid: { + location: { + lat: 68.15180202014744, + lon: 33.46390150487423, + }, + count: 1, + }, + avg_of_FlightDelayMin: { + value: 3, + }, + }, + ], + }, + }, + ], + }, + }, +}; + +it('Should convert elasticsearch aggregation response into feature collection of lines', () => { + const geoJson = convertToLines(esResponse); + expect(geoJson.featureCollection.features.length).toBe(1); + expect(geoJson.featureCollection.features[0]).toEqual({ + geometry: { + coordinates: [ + [33.46390150487423, 68.15180202014744], + [10.39269994944334, 43.68389896117151], + ], + type: 'LineString', + }, + id: '10.39269994944334,43.68389896117151,4/9/3', + properties: { + avg_of_FlightDelayMin: 3, + doc_count: 1, + terms_of_Carrier: 'ES-Air', + }, + type: 'Feature', + }); +}); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js index 176ab62baf98c..53536b11aaca6 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js @@ -20,7 +20,6 @@ import { i18n } from '@kbn/i18n'; import { SOURCE_DATA_ID_ORIGIN, ES_PEW_PEW, COUNT_PROP_NAME } from '../../../../common/constants'; import { getDataSourceLabel } from '../../../../common/i18n_getters'; import { convertToLines } from './convert_to_lines'; -import { AggConfigs, Schemas } from 'ui/agg_types'; import { AbstractESAggSource } from '../es_agg_source'; import { DynamicStyleProperty } from '../../styles/vector/properties/dynamic_style_property'; import { COLOR_GRADIENTS } from '../../styles/color_utils'; @@ -28,8 +27,6 @@ import { indexPatterns } from '../../../../../../../../src/plugins/data/public'; const MAX_GEOTILE_LEVEL = 29; -const aggSchemas = new Schemas([AbstractESAggSource.METRIC_SCHEMA_CONFIG]); - export class ESPewPewSource extends AbstractESAggSource { static type = ES_PEW_PEW; static title = i18n.translate('xpack.maps.source.pewPewTitle', { @@ -170,9 +167,6 @@ export class ESPewPewSource extends AbstractESAggSource { async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { const indexPattern = await this.getIndexPattern(); - const metricAggConfigs = this.createMetricAggConfigs(); - const aggConfigs = new AggConfigs(indexPattern, metricAggConfigs, aggSchemas.all); - const searchSource = await this._makeSearchSource(searchFilters, 0); searchSource.setField('aggs', { destSplit: { @@ -199,7 +193,7 @@ export class ESPewPewSource extends AbstractESAggSource { field: this._descriptor.sourceGeoField, }, }, - ...aggConfigs.toDsl(), + ...this.getValueAggsDsl(indexPattern), }, }, }, diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index 288dd117da137..3533282436139 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -28,31 +28,7 @@ import { loadIndexSettings } from './load_index_settings'; import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants'; import { ESDocField } from '../../fields/es_doc_field'; - -function getField(indexPattern, fieldName) { - const field = indexPattern.fields.getByName(fieldName); - if (!field) { - throw new Error( - i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', { - defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`, - values: { fieldName, indexPatternTitle: indexPattern.title }, - }) - ); - } - return field; -} - -function addFieldToDSL(dsl, field) { - return !field.scripted - ? { ...dsl, field: field.name } - : { - ...dsl, - script: { - source: field.script, - lang: field.lang, - }, - }; -} +import { getField, addFieldToDSL } from '../../util/es_agg_utils'; function getDocValueAndSourceFields(indexPattern, fieldNames) { const docValueFields = []; diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js index d78d3038f870d..782f2845ceeff 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_source.js @@ -18,7 +18,7 @@ import { AggConfigs } from 'ui/agg_types'; import { i18n } from '@kbn/i18n'; import uuid from 'uuid/v4'; import { copyPersistentState } from '../../reducers/util'; -import { ES_GEO_FIELD_TYPE, METRIC_TYPE } from '../../../common/constants'; +import { ES_GEO_FIELD_TYPE, AGG_TYPE } from '../../../common/constants'; import { DataRequestAbortError } from '../util/data_request'; import { expandToTileBoundaries } from './es_geo_grid_source/geo_tile_utils'; @@ -270,7 +270,7 @@ export class AbstractESSource extends AbstractVectorSource { // Do not use field formatters for counting metrics if ( metricField && - (metricField.type === METRIC_TYPE.COUNT || metricField.type === METRIC_TYPE.UNIQUE_COUNT) + (metricField.type === AGG_TYPE.COUNT || metricField.type === AGG_TYPE.UNIQUE_COUNT) ) { return null; } @@ -347,13 +347,16 @@ export class AbstractESSource extends AbstractVectorSource { } getValueSuggestions = async (fieldName, query) => { - if (!fieldName) { + // fieldName could be an aggregation so it needs to be unpacked to expose raw field. + const metricField = this.getMetricFields().find(field => field.getName() === fieldName); + const realFieldName = metricField ? metricField.getESDocFieldName() : fieldName; + if (!realFieldName) { return []; } try { const indexPattern = await this.getIndexPattern(); - const field = indexPattern.fields.getByName(fieldName); + const field = indexPattern.fields.getByName(realFieldName); return await autocompleteService.getValueSuggestions({ indexPattern, field, diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js index 7d7a2e159d128..9cc2919404a94 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js @@ -6,46 +6,25 @@ import _ from 'lodash'; -import { AggConfigs, Schemas } from 'ui/agg_types'; import { i18n } from '@kbn/i18n'; -import { - COUNT_PROP_LABEL, - DEFAULT_MAX_BUCKETS_LIMIT, - FIELD_ORIGIN, - METRIC_TYPE, -} from '../../../common/constants'; +import { DEFAULT_MAX_BUCKETS_LIMIT, FIELD_ORIGIN, AGG_TYPE } from '../../../common/constants'; import { ESDocField } from '../fields/es_doc_field'; import { AbstractESAggSource, AGG_DELIMITER } from './es_agg_source'; +import { getField, addFieldToDSL, extractPropertiesFromBucket } from '../util/es_agg_utils'; const TERMS_AGG_NAME = 'join'; const FIELD_NAME_PREFIX = '__kbnjoin__'; const GROUP_BY_DELIMITER = '_groupby_'; +const TERMS_BUCKET_KEYS_TO_IGNORE = ['key', 'doc_count']; -const aggSchemas = new Schemas([ - AbstractESAggSource.METRIC_SCHEMA_CONFIG, - { - group: 'buckets', - name: 'segment', - title: 'Terms', - aggFilter: 'terms', - min: 1, - max: 1, - }, -]); - -export function extractPropertiesMap(rawEsData, propertyNames, countPropertyName) { +export function extractPropertiesMap(rawEsData, countPropertyName) { const propertiesMap = new Map(); _.get(rawEsData, ['aggregations', TERMS_AGG_NAME, 'buckets'], []).forEach(termBucket => { - const properties = {}; + const properties = extractPropertiesFromBucket(termBucket, TERMS_BUCKET_KEYS_TO_IGNORE); if (countPropertyName) { properties[countPropertyName] = termBucket.doc_count; } - propertyNames.forEach(propertyName => { - if (_.has(termBucket, [propertyName, 'value'])) { - properties[propertyName] = _.get(termBucket, [propertyName, 'value']); - } - }); propertiesMap.set(termBucket.key.toString(), properties); }); return propertiesMap; @@ -90,15 +69,27 @@ export class ESTermSource extends AbstractESAggSource { formatMetricKey(aggType, fieldName) { const metricKey = - aggType !== METRIC_TYPE.COUNT ? `${aggType}${AGG_DELIMITER}${fieldName}` : aggType; + aggType !== AGG_TYPE.COUNT ? `${aggType}${AGG_DELIMITER}${fieldName}` : aggType; return `${FIELD_NAME_PREFIX}${metricKey}${GROUP_BY_DELIMITER}${ this._descriptor.indexPatternTitle }.${this._termField.getName()}`; } formatMetricLabel(type, fieldName) { - const metricLabel = type !== METRIC_TYPE.COUNT ? `${type} ${fieldName}` : COUNT_PROP_LABEL; - return `${metricLabel} of ${this._descriptor.indexPatternTitle}:${this._termField.getName()}`; + switch (type) { + case AGG_TYPE.COUNT: + return i18n.translate('xpack.maps.source.esJoin.countLabel', { + defaultMessage: `Count of {indexPatternTitle}`, + values: { indexPatternTitle: this._descriptor.indexPatternTitle }, + }); + case AGG_TYPE.TERMS: + return i18n.translate('xpack.maps.source.esJoin.topTermLabel', { + defaultMessage: `Top {fieldName}`, + values: { fieldName }, + }); + default: + return `${type} ${fieldName}`; + } } async getPropertiesMap(searchFilters, leftSourceName, leftFieldName, registerCancelCallback) { @@ -108,9 +99,14 @@ export class ESTermSource extends AbstractESAggSource { const indexPattern = await this.getIndexPattern(); const searchSource = await this._makeSearchSource(searchFilters, 0); - const configStates = this._makeAggConfigs(); - const aggConfigs = new AggConfigs(indexPattern, configStates, aggSchemas.all); - searchSource.setField('aggs', aggConfigs.toDsl()); + const termsField = getField(indexPattern, this._termField.getName()); + const termsAgg = { size: DEFAULT_MAX_BUCKETS_LIMIT }; + searchSource.setField('aggs', { + [TERMS_AGG_NAME]: { + terms: addFieldToDSL(termsAgg, termsField), + aggs: { ...this.getValueAggsDsl(indexPattern) }, + }, + }); const rawEsData = await this._runEsQuery({ requestId: this.getId(), @@ -120,19 +116,9 @@ export class ESTermSource extends AbstractESAggSource { requestDescription: this._getRequestDescription(leftSourceName, leftFieldName), }); - const metricPropertyNames = configStates - .filter(configState => { - return configState.schema === 'metric' && configState.type !== METRIC_TYPE.COUNT; - }) - .map(configState => { - return configState.id; - }); - const countConfigState = configStates.find(configState => { - return configState.type === METRIC_TYPE.COUNT; - }); - const countPropertyName = _.get(countConfigState, 'id'); + const countPropertyName = this.formatMetricKey(AGG_TYPE.COUNT); return { - propertiesMap: extractPropertiesMap(rawEsData, metricPropertyNames, countPropertyName), + propertiesMap: extractPropertiesMap(rawEsData, countPropertyName), }; } @@ -164,23 +150,6 @@ export class ESTermSource extends AbstractESAggSource { }); } - _makeAggConfigs() { - const metricAggConfigs = this.createMetricAggConfigs(); - return [ - ...metricAggConfigs, - { - id: TERMS_AGG_NAME, - enabled: true, - type: 'terms', - schema: 'segment', - params: { - field: this._termField.getName(), - size: DEFAULT_MAX_BUCKETS_LIMIT, - }, - }, - ]; - } - async getDisplayName() { //no need to localize. this is never rendered. return `es_table ${this._descriptor.indexPatternId}`; diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js index ffaaf2d705b5c..39cc301d458cb 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js @@ -8,9 +8,6 @@ import { ESTermSource, extractPropertiesMap } from './es_term_source'; jest.mock('ui/new_platform'); jest.mock('../vector_layer', () => {}); -jest.mock('ui/agg_types', () => ({ - Schemas: function() {}, -})); jest.mock('ui/timefilter', () => {}); const indexPatternTitle = 'myIndex'; @@ -44,7 +41,7 @@ describe('getMetricFields', () => { expect(metrics[0].getAggType()).toEqual('count'); expect(metrics[0].getName()).toEqual('__kbnjoin__count_groupby_myIndex.myTermField'); - expect(await metrics[0].getLabel()).toEqual('count of myIndex:myTermField'); + expect(await metrics[0].getLabel()).toEqual('Count of myIndex'); }); it('should remove incomplete metric configurations', async () => { @@ -65,84 +62,13 @@ describe('getMetricFields', () => { expect(metrics[1].getAggType()).toEqual('count'); expect(metrics[1].getName()).toEqual('__kbnjoin__count_groupby_myIndex.myTermField'); - expect(await metrics[1].getLabel()).toEqual('count of myIndex:myTermField'); - }); -}); - -describe('_makeAggConfigs', () => { - describe('no metrics', () => { - let aggConfigs; - beforeAll(() => { - const source = new ESTermSource({ - indexPatternTitle: indexPatternTitle, - term: termFieldName, - }); - aggConfigs = source._makeAggConfigs(); - }); - - it('should make default "count" metric agg config', () => { - expect(aggConfigs.length).toBe(2); - expect(aggConfigs[0]).toEqual({ - id: '__kbnjoin__count_groupby_myIndex.myTermField', - enabled: true, - type: 'count', - schema: 'metric', - params: {}, - }); - }); - - it('should make "terms" buckets agg config', () => { - expect(aggConfigs.length).toBe(2); - expect(aggConfigs[1]).toEqual({ - id: 'join', - enabled: true, - type: 'terms', - schema: 'segment', - params: { - field: termFieldName, - size: 10000, - }, - }); - }); - }); - - describe('metrics', () => { - let aggConfigs; - beforeAll(() => { - const source = new ESTermSource({ - indexPatternTitle: indexPatternTitle, - term: 'myTermField', - metrics: metricExamples, - }); - aggConfigs = source._makeAggConfigs(); - }); - - it('should ignore invalid metrics configs', () => { - expect(aggConfigs.length).toBe(3); - }); - - it('should make agg config for each valid metric', () => { - expect(aggConfigs[0]).toEqual({ - id: '__kbnjoin__sum_of_myFieldGettingSummed_groupby_myIndex.myTermField', - enabled: true, - type: 'sum', - schema: 'metric', - params: { - field: sumFieldName, - }, - }); - expect(aggConfigs[1]).toEqual({ - id: '__kbnjoin__count_groupby_myIndex.myTermField', - enabled: true, - type: 'count', - schema: 'metric', - params: {}, - }); - }); + expect(await metrics[1].getLabel()).toEqual('Count of myIndex'); }); }); describe('extractPropertiesMap', () => { + const minPropName = + '__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr'; const responseWithNumberTypes = { aggregations: { join: { @@ -150,14 +76,14 @@ describe('extractPropertiesMap', () => { { key: 109, doc_count: 1130, - '__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr': { + [minPropName]: { value: 36, }, }, { key: 62, doc_count: 448, - '__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr': { + [minPropName]: { value: 0, }, }, @@ -166,11 +92,10 @@ describe('extractPropertiesMap', () => { }, }; const countPropName = '__kbnjoin__count_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr'; - const minPropName = - '__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr'; + let propertiesMap; beforeAll(() => { - propertiesMap = extractPropertiesMap(responseWithNumberTypes, [minPropName], countPropName); + propertiesMap = extractPropertiesMap(responseWithNumberTypes, countPropName); }); it('should create key for each join term', () => { diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_size_property.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_size_property.js index e137e15730827..dfc5c530cc90f 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_size_property.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_size_property.js @@ -43,16 +43,8 @@ function getSymbolSizeIcons() { } export class DynamicSizeProperty extends DynamicStyleProperty { - constructor( - options, - styleName, - field, - getFieldMeta, - getFieldFormatter, - getValueSuggestions, - isSymbolizedAsIcon - ) { - super(options, styleName, field, getFieldMeta, getFieldFormatter, getValueSuggestions); + constructor(options, styleName, field, getFieldMeta, getFieldFormatter, isSymbolizedAsIcon) { + super(options, styleName, field, getFieldMeta, getFieldFormatter); this._isSymbolizedAsIcon = isSymbolizedAsIcon; } diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js index ef19e9b23b10d..af78c4c0e461e 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js @@ -13,21 +13,22 @@ import React from 'react'; import { OrdinalLegend } from './components/ordinal_legend'; import { CategoricalLegend } from './components/categorical_legend'; import { OrdinalFieldMetaOptionsPopover } from '../components/ordinal_field_meta_options_popover'; +import { ESAggMetricField } from '../../../fields/es_agg_field'; export class DynamicStyleProperty extends AbstractStyleProperty { static type = STYLE_TYPE.DYNAMIC; - constructor(options, styleName, field, getFieldMeta, getFieldFormatter, source) { + constructor(options, styleName, field, getFieldMeta, getFieldFormatter) { super(options, styleName); this._field = field; this._getFieldMeta = getFieldMeta; this._getFieldFormatter = getFieldFormatter; - this._source = source; } getValueSuggestions = query => { const fieldName = this.getFieldName(); - return this._source && fieldName ? this._source.getValueSuggestions(fieldName, query) : []; + const fieldSource = this.getFieldSource(); + return fieldSource && fieldName ? fieldSource.getValueSuggestions(fieldName, query) : []; }; getFieldMeta() { @@ -38,6 +39,10 @@ export class DynamicStyleProperty extends AbstractStyleProperty { return this._field; } + getFieldSource() { + return this._field ? this._field.getSource() : null; + } + getFieldName() { return this._field ? this._field.getName() : ''; } @@ -180,9 +185,10 @@ export class DynamicStyleProperty extends AbstractStyleProperty { } _pluckOrdinalStyleMetaFromFieldMetaData(fieldMetaData) { - const realFieldName = this._field.getESDocFieldName - ? this._field.getESDocFieldName() - : this._field.getName(); + const realFieldName = + this._field instanceof ESAggMetricField + ? this._field.getESDocFieldName() + : this._field.getName(); const stats = fieldMetaData[realFieldName]; if (!stats) { return null; @@ -203,12 +209,15 @@ export class DynamicStyleProperty extends AbstractStyleProperty { } _pluckCategoricalStyleMetaFromFieldMetaData(fieldMetaData) { - const name = this.getField().getName(); - if (!fieldMetaData[name] || !fieldMetaData[name].buckets) { + const realFieldName = + this._field instanceof ESAggMetricField + ? this._field.getESDocFieldName() + : this._field.getName(); + if (!fieldMetaData[realFieldName] || !fieldMetaData[realFieldName].buckets) { return null; } - const ordered = fieldMetaData[name].buckets.map(bucket => { + const ordered = fieldMetaData[realFieldName].buckets.map(bucket => { return { key: bucket.key, count: bucket.doc_count, diff --git a/x-pack/legacy/plugins/maps/public/layers/styles/vector/vector_style.js b/x-pack/legacy/plugins/maps/public/layers/styles/vector/vector_style.js index 62651fdd702d6..053aa114d94ae 100644 --- a/x-pack/legacy/plugins/maps/public/layers/styles/vector/vector_style.js +++ b/x-pack/legacy/plugins/maps/public/layers/styles/vector/vector_style.js @@ -625,7 +625,6 @@ export class VectorStyle extends AbstractStyle { field, this._getFieldMeta, this._getFieldFormatter, - this._source, isSymbolizedAsIcon ); } else { @@ -645,8 +644,7 @@ export class VectorStyle extends AbstractStyle { styleName, field, this._getFieldMeta, - this._getFieldFormatter, - this._source + this._getFieldFormatter ); } else { throw new Error(`${descriptor} not implemented`); @@ -678,8 +676,7 @@ export class VectorStyle extends AbstractStyle { VECTOR_STYLES.LABEL_TEXT, field, this._getFieldMeta, - this._getFieldFormatter, - this._source + this._getFieldFormatter ); } else { throw new Error(`${descriptor} not implemented`); @@ -698,8 +695,7 @@ export class VectorStyle extends AbstractStyle { VECTOR_STYLES.ICON, field, this._getFieldMeta, - this._getFieldFormatter, - this._source + this._getFieldFormatter ); } else { throw new Error(`${descriptor} not implemented`); diff --git a/x-pack/legacy/plugins/maps/public/layers/tooltips/es_aggmetric_tooltip_property.js b/x-pack/legacy/plugins/maps/public/layers/tooltips/es_aggmetric_tooltip_property.js index 7cfb60910c155..229c84fe234bd 100644 --- a/x-pack/legacy/plugins/maps/public/layers/tooltips/es_aggmetric_tooltip_property.js +++ b/x-pack/legacy/plugins/maps/public/layers/tooltips/es_aggmetric_tooltip_property.js @@ -5,7 +5,7 @@ */ import { ESTooltipProperty } from './es_tooltip_property'; -import { METRIC_TYPE } from '../../../common/constants'; +import { AGG_TYPE } from '../../../common/constants'; export class ESAggMetricTooltipProperty extends ESTooltipProperty { constructor(propertyKey, propertyName, rawValue, indexPattern, metricField) { @@ -22,8 +22,8 @@ export class ESAggMetricTooltipProperty extends ESTooltipProperty { return '-'; } if ( - this._metricField.getAggType() === METRIC_TYPE.COUNT || - this._metricField.getAggType() === METRIC_TYPE.UNIQUE_COUNT + this._metricField.getAggType() === AGG_TYPE.COUNT || + this._metricField.getAggType() === AGG_TYPE.UNIQUE_COUNT ) { return this._rawValue; } diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.ts b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.ts new file mode 100644 index 0000000000000..201d6907981a2 --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { extractPropertiesFromBucket } from './es_agg_utils'; + +describe('extractPropertiesFromBucket', () => { + test('Should ignore specified keys', () => { + const properties = extractPropertiesFromBucket({ key: '4/4/6' }, ['key']); + expect(properties).toEqual({}); + }); + + test('Should extract metric aggregation values', () => { + const properties = extractPropertiesFromBucket({ avg_of_bytes: { value: 5359 } }); + expect(properties).toEqual({ + avg_of_bytes: 5359, + }); + }); + + test('Should extract bucket aggregation values', () => { + const properties = extractPropertiesFromBucket({ + 'terms_of_machine.os.keyword': { + buckets: [ + { + key: 'win xp', + doc_count: 16, + }, + ], + }, + }); + expect(properties).toEqual({ + 'terms_of_machine.os.keyword': 'win xp', + }); + }); +}); diff --git a/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts new file mode 100644 index 0000000000000..7af176acfaf46 --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { i18n } from '@kbn/i18n'; +import _ from 'lodash'; +import { IndexPattern, IFieldType } from '../../../../../../../src/plugins/data/public'; + +export function getField(indexPattern: IndexPattern, fieldName: string) { + const field = indexPattern.fields.getByName(fieldName); + if (!field) { + throw new Error( + i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', { + defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`, + values: { fieldName, indexPatternTitle: indexPattern.title }, + }) + ); + } + return field; +} + +export function addFieldToDSL(dsl: object, field: IFieldType) { + return !field.scripted + ? { ...dsl, field: field.name } + : { + ...dsl, + script: { + source: field.script, + lang: field.lang, + }, + }; +} + +export function extractPropertiesFromBucket(bucket: any, ignoreKeys: string[] = []) { + const properties: Record = {}; + for (const key in bucket) { + if (ignoreKeys.includes(key) || !bucket.hasOwnProperty(key)) { + continue; + } + + if (_.has(bucket[key], 'value')) { + properties[key] = bucket[key].value; + } else if (_.has(bucket[key], 'buckets')) { + properties[key] = _.get(bucket[key], 'buckets[0].key'); + } else { + properties[key] = bucket[key]; + } + } + return properties; +} diff --git a/x-pack/legacy/plugins/maps/public/layers/util/is_metric_countable.js b/x-pack/legacy/plugins/maps/public/layers/util/is_metric_countable.js index 54d8794b1e3cf..69ccb8890d10c 100644 --- a/x-pack/legacy/plugins/maps/public/layers/util/is_metric_countable.js +++ b/x-pack/legacy/plugins/maps/public/layers/util/is_metric_countable.js @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { METRIC_TYPE } from '../../../common/constants'; +import { AGG_TYPE } from '../../../common/constants'; export function isMetricCountable(aggType) { - return [METRIC_TYPE.COUNT, METRIC_TYPE.SUM, METRIC_TYPE.UNIQUE_COUNT].includes(aggType); + return [AGG_TYPE.COUNT, AGG_TYPE.SUM, AGG_TYPE.UNIQUE_COUNT].includes(aggType); } diff --git a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js index 1698d52ea4406..e1a30c8aef1d3 100644 --- a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js +++ b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js @@ -365,8 +365,10 @@ export class VectorLayer extends AbstractLayer { onLoadError, registerCancelCallback, dataFilters, + isRequestStillActive, }) { - const requestToken = Symbol(`layer-${this.getId()}-${SOURCE_DATA_ID_ORIGIN}`); + const dataRequestId = SOURCE_DATA_ID_ORIGIN; + const requestToken = Symbol(`layer-${this.getId()}-${dataRequestId}`); const searchFilters = this._getSearchFilters(dataFilters); const prevDataRequest = this.getSourceDataRequest(); const canSkipFetch = await canSkipSourceUpdate({ @@ -382,22 +384,25 @@ export class VectorLayer extends AbstractLayer { } try { - startLoading(SOURCE_DATA_ID_ORIGIN, requestToken, searchFilters); + startLoading(dataRequestId, requestToken, searchFilters); const layerName = await this.getDisplayName(); const { data: sourceFeatureCollection, meta } = await this._source.getGeoJsonWithMeta( layerName, searchFilters, - registerCancelCallback.bind(null, requestToken) + registerCancelCallback.bind(null, requestToken), + () => { + return isRequestStillActive(dataRequestId, requestToken); + } ); const layerFeatureCollection = assignFeatureIds(sourceFeatureCollection); - stopLoading(SOURCE_DATA_ID_ORIGIN, requestToken, layerFeatureCollection, meta); + stopLoading(dataRequestId, requestToken, layerFeatureCollection, meta); return { refreshed: true, featureCollection: layerFeatureCollection, }; } catch (error) { if (!(error instanceof DataRequestAbortError)) { - onLoadError(SOURCE_DATA_ID_ORIGIN, requestToken, error.message); + onLoadError(dataRequestId, requestToken, error.message); } return { refreshed: false, diff --git a/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js b/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js index d1048a759beca..4074344916390 100644 --- a/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js +++ b/x-pack/legacy/plugins/maps/public/selectors/map_selectors.js @@ -125,6 +125,21 @@ export const getRefreshConfig = ({ map }) => { export const getRefreshTimerLastTriggeredAt = ({ map }) => map.mapState.refreshTimerLastTriggeredAt; +function getLayerDescriptor(state = {}, layerId) { + const layerListRaw = getLayerListRaw(state); + return layerListRaw.find(layer => layer.id === layerId); +} + +export function getDataRequestDescriptor(state = {}, layerId, dataId) { + const layerDescriptor = getLayerDescriptor(state, layerId); + if (!layerDescriptor || !layerDescriptor.__dataRequests) { + return; + } + return _.get(layerDescriptor, '__dataRequests', []).find(dataRequest => { + return dataRequest.dataId === dataId; + }); +} + export const getDataFilters = createSelector( getMapExtent, getMapBuffer, diff --git a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js index 9317d3c6c3e07..e3092abb5d34e 100644 --- a/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js +++ b/x-pack/legacy/plugins/ml/server/client/elasticsearch_ml.js @@ -450,6 +450,18 @@ export const elasticsearchJsPlugin = (Client, config, components) => { method: 'POST', }); + ml.records = ca({ + url: { + fmt: '/_ml/anomaly_detectors/<%=jobId%>/results/records', + req: { + jobId: { + type: 'string', + }, + }, + }, + method: 'POST', + }); + ml.buckets = ca({ urls: [ { diff --git a/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.ts b/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.ts index 927646e4f0acc..99dbdec9e945b 100644 --- a/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.ts +++ b/x-pack/legacy/plugins/ml/server/routes/anomaly_detectors.ts @@ -376,11 +376,57 @@ export function jobRoutes({ xpackMainPlugin, router }: RouteInitialization) { }) ); + /** + * @apiGroup AnomalyDetectors + * + * @api {post} /api/ml/anomaly_detectors/:jobId/results/records Retrieves anomaly records for a job. + * @apiName GetRecords + * @apiDescription Retrieves anomaly records for a job. + * + * @apiParam {String} jobId Job ID. + * + * @apiSuccess {Number} count + * @apiSuccess {Object[]} records + */ + router.post( + { + path: '/api/ml/anomaly_detectors/{jobId}/results/records', + validate: { + params: schema.object({ + jobId: schema.string(), + }), + body: schema.object({ + desc: schema.maybe(schema.boolean()), + end: schema.maybe(schema.string()), + exclude_interim: schema.maybe(schema.boolean()), + 'page.from': schema.maybe(schema.number()), + 'page.size': schema.maybe(schema.number()), + record_score: schema.maybe(schema.number()), + sort: schema.maybe(schema.string()), + start: schema.maybe(schema.string()), + }), + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const results = await context.ml!.mlClient.callAsCurrentUser('ml.records', { + jobId: request.params.jobId, + ...request.body, + }); + return response.ok({ + body: results, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + /** * @apiGroup AnomalyDetectors * * @api {post} /api/ml/anomaly_detectors/:jobId/results/buckets Obtain bucket scores for the specified job ID - * @apiName GetOverallBuckets + * @apiName GetBuckets * @apiDescription The get buckets API presents a chronological view of the records, grouped by bucket. * * @apiParam {String} jobId Job ID. diff --git a/x-pack/legacy/plugins/ml/server/routes/apidoc.json b/x-pack/legacy/plugins/ml/server/routes/apidoc.json index 946e3bd71d6c3..c5aa3e4d792fd 100644 --- a/x-pack/legacy/plugins/ml/server/routes/apidoc.json +++ b/x-pack/legacy/plugins/ml/server/routes/apidoc.json @@ -31,6 +31,8 @@ "DeleteAnomalyDetectorsJob", "ValidateAnomalyDetector", "ForecastAnomalyDetector", + "GetRecords", + "GetBuckets", "GetOverallBuckets", "GetCategories", "FileDataVisualizer", diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rules.test.tsx b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rules.test.tsx index b369d3a50730d..242d715e20f77 100644 --- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rules.test.tsx +++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rules.test.tsx @@ -5,9 +5,8 @@ */ import { renderHook, act } from '@testing-library/react-hooks'; -import { useRules, ReturnRules } from './use_rules'; +import { useRules, UseRules, ReturnRules } from './use_rules'; import * as api from './api'; -import { PaginationOptions, FilterOptions } from '.'; jest.mock('./api'); @@ -17,55 +16,40 @@ describe('useRules', () => { }); test('init', async () => { await act(async () => { - const { result, waitForNextUpdate } = renderHook< - [PaginationOptions, FilterOptions], - ReturnRules - >(props => - useRules( - { + const { result, waitForNextUpdate } = renderHook(props => + useRules({ + pagination: { page: 1, perPage: 10, total: 100, }, - { + filterOptions: { filter: '', sortField: 'created_at', sortOrder: 'desc', - } - ) + }, + }) ); await waitForNextUpdate(); - expect(result.current).toEqual([ - true, - { - data: [], - page: 1, - perPage: 20, - total: 0, - }, - null, - ]); + expect(result.current).toEqual([true, null, result.current[2]]); }); }); test('fetch rules', async () => { await act(async () => { - const { result, waitForNextUpdate } = renderHook< - [PaginationOptions, FilterOptions], - ReturnRules - >(() => - useRules( - { + const { result, waitForNextUpdate } = renderHook(() => + useRules({ + pagination: { page: 1, perPage: 10, total: 100, }, - { + filterOptions: { filter: '', sortField: 'created_at', sortOrder: 'desc', - } - ) + }, + }) ); await waitForNextUpdate(); await waitForNextUpdate(); @@ -148,22 +132,19 @@ describe('useRules', () => { test('re-fetch rules', async () => { const spyOnfetchRules = jest.spyOn(api, 'fetchRules'); await act(async () => { - const { result, waitForNextUpdate } = renderHook< - [PaginationOptions, FilterOptions], - ReturnRules - >(id => - useRules( - { + const { result, waitForNextUpdate } = renderHook(id => + useRules({ + pagination: { page: 1, perPage: 10, total: 100, }, - { + filterOptions: { filter: '', sortField: 'created_at', sortOrder: 'desc', - } - ) + }, + }) ); await waitForNextUpdate(); await waitForNextUpdate(); @@ -178,37 +159,37 @@ describe('useRules', () => { test('fetch rules if props changes', async () => { const spyOnfetchRules = jest.spyOn(api, 'fetchRules'); await act(async () => { - const { rerender, waitForNextUpdate } = renderHook< - [PaginationOptions, FilterOptions], - ReturnRules - >(args => useRules(args[0], args[1]), { - initialProps: [ - { - page: 1, - perPage: 10, - total: 100, - }, - { - filter: '', - sortField: 'created_at', - sortOrder: 'desc', + const { rerender, waitForNextUpdate } = renderHook( + args => useRules(args), + { + initialProps: { + pagination: { + page: 1, + perPage: 10, + total: 100, + }, + filterOptions: { + filter: '', + sortField: 'created_at', + sortOrder: 'desc', + }, }, - ], - }); + } + ); await waitForNextUpdate(); await waitForNextUpdate(); - rerender([ - { + rerender({ + pagination: { page: 1, perPage: 10, total: 100, }, - { + filterOptions: { filter: 'hello world', sortField: 'created_at', sortOrder: 'desc', }, - ]); + }); await waitForNextUpdate(); expect(spyOnfetchRules).toHaveBeenCalledTimes(2); }); diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rules.tsx b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rules.tsx index 301a68dc6f445..d05d59d15802d 100644 --- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rules.tsx +++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rules.tsx @@ -4,16 +4,27 @@ * you may not use this file except in compliance with the Elastic License. */ +import { noop } from 'lodash/fp'; import { useEffect, useState, useRef } from 'react'; -import { FetchRulesResponse, FilterOptions, PaginationOptions } from './types'; +import { FetchRulesResponse, FilterOptions, PaginationOptions, Rule } from './types'; import { useStateToaster } from '../../../components/toasters'; import { fetchRules } from './api'; import { errorToToaster } from '../../../components/ml/api/error_to_toaster'; import * as i18n from './translations'; -type Func = () => void; -export type ReturnRules = [boolean, FetchRulesResponse, Func | null]; +export type ReturnRules = [ + boolean, + FetchRulesResponse | null, + (refreshPrePackagedRule?: boolean) => void +]; + +export interface UseRules { + pagination: PaginationOptions; + filterOptions: FilterOptions; + refetchPrePackagedRulesStatus?: () => void; + dispatchRulesInReducer?: (rules: Rule[]) => void; +} /** * Hook for using the list of Rules from the Detection Engine API @@ -21,17 +32,14 @@ export type ReturnRules = [boolean, FetchRulesResponse, Func | null]; * @param pagination desired pagination options (e.g. page/perPage) * @param filterOptions desired filters (e.g. filter/sortField/sortOrder) */ -export const useRules = ( - pagination: PaginationOptions, - filterOptions: FilterOptions -): ReturnRules => { - const [rules, setRules] = useState({ - page: 1, - perPage: 20, - total: 0, - data: [], - }); - const reFetchRules = useRef(null); +export const useRules = ({ + pagination, + filterOptions, + refetchPrePackagedRulesStatus, + dispatchRulesInReducer, +}: UseRules): ReturnRules => { + const [rules, setRules] = useState(null); + const reFetchRules = useRef<(refreshPrePackagedRule?: boolean) => void>(noop); const [loading, setLoading] = useState(true); const [, dispatchToaster] = useStateToaster(); @@ -50,10 +58,16 @@ export const useRules = ( if (isSubscribed) { setRules(fetchRulesResult); + if (dispatchRulesInReducer != null) { + dispatchRulesInReducer(fetchRulesResult.data); + } } } catch (error) { if (isSubscribed) { errorToToaster({ title: i18n.RULE_FETCH_FAILURE, error, dispatchToaster }); + if (dispatchRulesInReducer != null) { + dispatchRulesInReducer([]); + } } } if (isSubscribed) { @@ -62,7 +76,12 @@ export const useRules = ( } fetchData(); - reFetchRules.current = fetchData.bind(null, true); + reFetchRules.current = (refreshPrePackagedRule: boolean = false) => { + fetchData(true); + if (refreshPrePackagedRule && refetchPrePackagedRulesStatus != null) { + refetchPrePackagedRulesStatus(); + } + }; return () => { isSubscribed = false; abortCtrl.abort(); @@ -76,6 +95,7 @@ export const useRules = ( filterOptions.tags?.sort().join(), filterOptions.showCustomRules, filterOptions.showElasticRules, + refetchPrePackagedRulesStatus, ]); return [loading, rules, reFetchRules.current]; diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_tags.test.tsx b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_tags.test.tsx index 4a796efa5b0cb..68f54b35754f6 100644 --- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_tags.test.tsx +++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_tags.test.tsx @@ -14,7 +14,7 @@ describe('useTags', () => { await act(async () => { const { result, waitForNextUpdate } = renderHook(() => useTags()); await waitForNextUpdate(); - expect(result.current).toEqual([true, []]); + expect(result.current).toEqual([true, [], result.current[2]]); }); }); @@ -23,7 +23,11 @@ describe('useTags', () => { const { result, waitForNextUpdate } = renderHook(() => useTags()); await waitForNextUpdate(); await waitForNextUpdate(); - expect(result.current).toEqual([false, ['elastic', 'love', 'quality', 'code']]); + expect(result.current).toEqual([ + false, + ['elastic', 'love', 'quality', 'code'], + result.current[2], + ]); }); }); }); diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_tags.tsx b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_tags.tsx index 196d4b1420561..5985200fa16ec 100644 --- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_tags.tsx +++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_tags.tsx @@ -4,13 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useEffect, useState } from 'react'; +import { noop } from 'lodash/fp'; +import { useEffect, useState, useRef } from 'react'; import { useStateToaster } from '../../../components/toasters'; import { fetchTags } from './api'; import { errorToToaster } from '../../../components/ml/api/error_to_toaster'; import * as i18n from './translations'; -export type ReturnTags = [boolean, string[]]; +export type ReturnTags = [boolean, string[], () => void]; /** * Hook for using the list of Tags from the Detection Engine API @@ -20,6 +21,7 @@ export const useTags = (): ReturnTags => { const [tags, setTags] = useState([]); const [loading, setLoading] = useState(true); const [, dispatchToaster] = useStateToaster(); + const reFetchTags = useRef<() => void>(noop); useEffect(() => { let isSubscribed = true; @@ -46,6 +48,7 @@ export const useTags = (): ReturnTags => { } fetchData(); + reFetchTags.current = fetchData; return () => { isSubscribed = false; @@ -53,5 +56,5 @@ export const useTags = (): ReturnTags => { }; }, []); - return [loading, tags]; + return [loading, tags, reFetchTags.current]; }; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts index 980575f1470a5..e2287e5eeeb3f 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -5,7 +5,6 @@ */ import { Rule, RuleError } from '../../../../../containers/detection_engine/rules'; -import { TableData } from '../../types'; export const mockRule = (id: string): Rule => ({ created_at: '2020-01-10T21:11:45.839Z', @@ -50,103 +49,3 @@ export const mockRules: Rule[] = [ mockRule('abe6c564-050d-45a5-aaf0-386c37dd1f61'), mockRule('63f06f34-c181-4b2d-af35-f2ace572a1ee'), ]; -export const mockTableData: TableData[] = [ - { - activate: true, - id: 'abe6c564-050d-45a5-aaf0-386c37dd1f61', - immutable: false, - isLoading: false, - risk_score: 21, - rule: { - href: '#/detections/rules/id/abe6c564-050d-45a5-aaf0-386c37dd1f61', - name: 'Home Grown!', - }, - rule_id: 'b5ba41ab-aaf3-4f43-971b-bdf9434ce0ea', - severity: 'low', - sourceRule: { - created_at: '2020-01-10T21:11:45.839Z', - created_by: 'elastic', - description: '24/7', - enabled: true, - false_positives: [], - filters: [], - from: 'now-300s', - id: 'abe6c564-050d-45a5-aaf0-386c37dd1f61', - immutable: false, - index: ['auditbeat-*'], - interval: '5m', - language: 'kuery', - max_signals: 100, - meta: { from: '0m' }, - name: 'Home Grown!', - output_index: '.siem-signals-default', - query: '', - references: [], - risk_score: 21, - rule_id: 'b5ba41ab-aaf3-4f43-971b-bdf9434ce0ea', - saved_id: "Garrett's IP", - severity: 'low', - tags: [], - threat: [], - timeline_id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', - timeline_title: 'Untitled timeline', - to: 'now', - type: 'saved_query', - updated_at: '2020-01-10T21:11:45.839Z', - updated_by: 'elastic', - version: 1, - }, - status: null, - statusDate: null, - tags: [], - }, - { - activate: true, - id: '63f06f34-c181-4b2d-af35-f2ace572a1ee', - immutable: false, - isLoading: false, - risk_score: 21, - rule: { - href: '#/detections/rules/id/63f06f34-c181-4b2d-af35-f2ace572a1ee', - name: 'Home Grown!', - }, - rule_id: 'b5ba41ab-aaf3-4f43-971b-bdf9434ce0ea', - severity: 'low', - sourceRule: { - created_at: '2020-01-10T21:11:45.839Z', - created_by: 'elastic', - description: '24/7', - enabled: true, - false_positives: [], - filters: [], - from: 'now-300s', - id: '63f06f34-c181-4b2d-af35-f2ace572a1ee', - immutable: false, - index: ['auditbeat-*'], - interval: '5m', - language: 'kuery', - max_signals: 100, - meta: { from: '0m' }, - name: 'Home Grown!', - output_index: '.siem-signals-default', - query: '', - references: [], - risk_score: 21, - rule_id: 'b5ba41ab-aaf3-4f43-971b-bdf9434ce0ea', - saved_id: "Garrett's IP", - severity: 'low', - tags: [], - threat: [], - timeline_id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', - timeline_title: 'Untitled timeline', - to: 'now', - type: 'saved_query', - updated_at: '2020-01-10T21:11:45.839Z', - updated_by: 'elastic', - version: 1, - }, - status: null, - statusDate: null, - tags: [], - }, -]; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/actions.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/actions.tsx index 6212c2067384d..a17fd34d1c344 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/actions.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/actions.tsx @@ -32,53 +32,58 @@ export const editRuleAction = (rule: Rule, history: H.History) => { export const duplicateRulesAction = async ( rules: Rule[], + ruleIds: string[], dispatch: React.Dispatch, dispatchToaster: Dispatch ) => { try { - const ruleIds = rules.map(r => r.id); - dispatch({ type: 'updateLoading', ids: ruleIds, isLoading: true }); - const duplicatedRules = await duplicateRules({ rules }); - dispatch({ type: 'refresh' }); - displaySuccessToast( - i18n.SUCCESSFULLY_DUPLICATED_RULES(duplicatedRules.length), - dispatchToaster - ); + dispatch({ type: 'loadingRuleIds', ids: ruleIds, actionType: 'duplicate' }); + const response = await duplicateRules({ rules }); + const { errors } = bucketRulesResponse(response); + if (errors.length > 0) { + displayErrorToast( + i18n.DUPLICATE_RULE_ERROR, + errors.map(e => e.error.message), + dispatchToaster + ); + } else { + displaySuccessToast(i18n.SUCCESSFULLY_DUPLICATED_RULES(ruleIds.length), dispatchToaster); + } + dispatch({ type: 'loadingRuleIds', ids: [], actionType: null }); } catch (e) { + dispatch({ type: 'loadingRuleIds', ids: [], actionType: null }); displayErrorToast(i18n.DUPLICATE_RULE_ERROR, [e.message], dispatchToaster); } }; -export const exportRulesAction = async (rules: Rule[], dispatch: React.Dispatch) => { - dispatch({ type: 'setExportPayload', exportPayload: rules }); +export const exportRulesAction = (exportRuleId: string[], dispatch: React.Dispatch) => { + dispatch({ type: 'exportRuleIds', ids: exportRuleId }); }; export const deleteRulesAction = async ( - ids: string[], + ruleIds: string[], dispatch: React.Dispatch, dispatchToaster: Dispatch, onRuleDeleted?: () => void ) => { try { - dispatch({ type: 'loading', isLoading: true }); - - const response = await deleteRules({ ids }); + dispatch({ type: 'loadingRuleIds', ids: ruleIds, actionType: 'delete' }); + const response = await deleteRules({ ids: ruleIds }); const { errors } = bucketRulesResponse(response); - - dispatch({ type: 'refresh' }); + dispatch({ type: 'loadingRuleIds', ids: [], actionType: null }); if (errors.length > 0) { displayErrorToast( - i18n.BATCH_ACTION_DELETE_SELECTED_ERROR(ids.length), + i18n.BATCH_ACTION_DELETE_SELECTED_ERROR(ruleIds.length), errors.map(e => e.error.message), dispatchToaster ); - } else { - // FP: See https://github.com/typescript-eslint/typescript-eslint/issues/1138#issuecomment-566929566 - onRuleDeleted?.(); // eslint-disable-line no-unused-expressions + } else if (onRuleDeleted) { + onRuleDeleted(); } } catch (e) { + dispatch({ type: 'loadingRuleIds', ids: [], actionType: null }); displayErrorToast( - i18n.BATCH_ACTION_DELETE_SELECTED_ERROR(ids.length), + i18n.BATCH_ACTION_DELETE_SELECTED_ERROR(ruleIds.length), [e.message], dispatchToaster ); @@ -96,7 +101,7 @@ export const enableRulesAction = async ( : i18n.BATCH_ACTION_DEACTIVATE_SELECTED_ERROR(ids.length); try { - dispatch({ type: 'updateLoading', ids, isLoading: true }); + dispatch({ type: 'loadingRuleIds', ids, actionType: enabled ? 'enable' : 'disable' }); const response = await enableRules({ ids, enabled }); const { rules, errors } = bucketRulesResponse(response); @@ -125,6 +130,6 @@ export const enableRulesAction = async ( } } catch (e) { displayErrorToast(errorTitle, [e.message], dispatchToaster); - dispatch({ type: 'updateLoading', ids, isLoading: false }); + dispatch({ type: 'loadingRuleIds', ids: [], actionType: null }); } }; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/batch_actions.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/batch_actions.tsx index 8a10d4f7100b9..a0942d7f6534a 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/batch_actions.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/batch_actions.tsx @@ -6,9 +6,7 @@ import { EuiContextMenuItem } from '@elastic/eui'; import React, { Dispatch } from 'react'; -import * as H from 'history'; import * as i18n from '../translations'; -import { TableData } from '../types'; import { Action } from './reducer'; import { deleteRulesAction, @@ -17,18 +15,37 @@ import { exportRulesAction, } from './actions'; import { ActionToaster } from '../../../../components/toasters'; +import { Rule } from '../../../../containers/detection_engine/rules'; -export const getBatchItems = ( - selectedState: TableData[], - dispatch: Dispatch, - dispatchToaster: Dispatch, - history: H.History, - closePopover: () => void -) => { - const containsEnabled = selectedState.some(v => v.activate); - const containsDisabled = selectedState.some(v => !v.activate); - const containsLoading = selectedState.some(v => v.isLoading); - const containsImmutable = selectedState.some(v => v.immutable); +interface GetBatchItems { + closePopover: () => void; + dispatch: Dispatch; + dispatchToaster: Dispatch; + loadingRuleIds: string[]; + reFetchRules: (refreshPrePackagedRule?: boolean) => void; + rules: Rule[]; + selectedRuleIds: string[]; +} + +export const getBatchItems = ({ + closePopover, + dispatch, + dispatchToaster, + loadingRuleIds, + reFetchRules, + rules, + selectedRuleIds, +}: GetBatchItems) => { + const containsEnabled = selectedRuleIds.some( + id => rules.find(r => r.id === id)?.enabled ?? false + ); + const containsDisabled = selectedRuleIds.some( + id => !rules.find(r => r.id === id)?.enabled ?? false + ); + const containsLoading = selectedRuleIds.some(id => loadingRuleIds.includes(id)); + const containsImmutable = selectedRuleIds.some( + id => rules.find(r => r.id === id)?.immutable ?? false + ); return [ { closePopover(); - const deactivatedIds = selectedState.filter(s => !s.activate).map(s => s.id); + const deactivatedIds = selectedRuleIds.filter( + id => !rules.find(r => r.id === id)?.enabled ?? false + ); await enableRulesAction(deactivatedIds, true, dispatch, dispatchToaster); }} > @@ -49,7 +68,9 @@ export const getBatchItems = ( disabled={containsLoading || !containsEnabled} onClick={async () => { closePopover(); - const activatedIds = selectedState.filter(s => s.activate).map(s => s.id); + const activatedIds = selectedRuleIds.filter( + id => rules.find(r => r.id === id)?.enabled ?? false + ); await enableRulesAction(activatedIds, false, dispatch, dispatchToaster); }} > @@ -58,11 +79,11 @@ export const getBatchItems = ( { + disabled={containsImmutable || containsLoading || selectedRuleIds.length === 0} + onClick={() => { closePopover(); - await exportRulesAction( - selectedState.map(s => s.sourceRule), + exportRulesAction( + rules.filter(r => selectedRuleIds.includes(r.id)).map(r => r.rule_id), dispatch ); }} @@ -72,14 +93,16 @@ export const getBatchItems = ( { closePopover(); await duplicateRulesAction( - selectedState.map(s => s.sourceRule), + rules.filter(r => selectedRuleIds.includes(r.id)), + selectedRuleIds, dispatch, dispatchToaster ); + reFetchRules(true); }} > {i18n.BATCH_ACTION_DUPLICATE_SELECTED} @@ -88,14 +111,11 @@ export const getBatchItems = ( key={i18n.BATCH_ACTION_DELETE_SELECTED} icon="trash" title={containsImmutable ? i18n.BATCH_ACTION_DELETE_SELECTED_IMMUTABLE : undefined} - disabled={containsLoading || selectedState.length === 0} + disabled={containsLoading || selectedRuleIds.length === 0} onClick={async () => { closePopover(); - await deleteRulesAction( - selectedState.map(({ sourceRule: { id } }) => id), - dispatch, - dispatchToaster - ); + await deleteRulesAction(selectedRuleIds, dispatch, dispatchToaster); + reFetchRules(true); }} > {i18n.BATCH_ACTION_DELETE_SELECTED} diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/columns.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/columns.tsx index d648854368c28..ff104f09d68ef 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/columns.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/columns.tsx @@ -15,72 +15,92 @@ import { } from '@elastic/eui'; import * as H from 'history'; import React, { Dispatch } from 'react'; + +import { Rule } from '../../../../containers/detection_engine/rules'; import { getEmptyTagValue } from '../../../../components/empty_value'; +import { FormattedDate } from '../../../../components/formatted_date'; +import { getRuleDetailsUrl } from '../../../../components/link_to/redirect_to_detection_engine'; +import { ActionToaster } from '../../../../components/toasters'; +import { TruncatableText } from '../../../../components/truncatable_text'; +import { getStatusColor } from '../components/rule_status/helpers'; +import { RuleSwitch } from '../components/rule_switch'; +import { SeverityBadge } from '../components/severity_badge'; +import * as i18n from '../translations'; import { deleteRulesAction, duplicateRulesAction, editRuleAction, exportRulesAction, } from './actions'; - import { Action } from './reducer'; -import { TableData } from '../types'; -import * as i18n from '../translations'; -import { FormattedDate } from '../../../../components/formatted_date'; -import { RuleSwitch } from '../components/rule_switch'; -import { SeverityBadge } from '../components/severity_badge'; -import { ActionToaster } from '../../../../components/toasters'; -import { getStatusColor } from '../components/rule_status/helpers'; -import { TruncatableText } from '../../../../components/truncatable_text'; const getActions = ( dispatch: React.Dispatch, dispatchToaster: Dispatch, - history: H.History + history: H.History, + reFetchRules: (refreshPrePackagedRule?: boolean) => void ) => [ { description: i18n.EDIT_RULE_SETTINGS, icon: 'visControls', name: i18n.EDIT_RULE_SETTINGS, - onClick: (rowItem: TableData) => editRuleAction(rowItem.sourceRule, history), - enabled: (rowItem: TableData) => !rowItem.sourceRule.immutable, + onClick: (rowItem: Rule) => editRuleAction(rowItem, history), + enabled: (rowItem: Rule) => !rowItem.immutable, }, { description: i18n.DUPLICATE_RULE, icon: 'copy', name: i18n.DUPLICATE_RULE, - onClick: (rowItem: TableData) => - duplicateRulesAction([rowItem.sourceRule], dispatch, dispatchToaster), + onClick: (rowItem: Rule) => { + duplicateRulesAction([rowItem], [rowItem.id], dispatch, dispatchToaster); + reFetchRules(true); + }, }, { description: i18n.EXPORT_RULE, icon: 'exportAction', name: i18n.EXPORT_RULE, - onClick: (rowItem: TableData) => exportRulesAction([rowItem.sourceRule], dispatch), - enabled: (rowItem: TableData) => !rowItem.immutable, + onClick: (rowItem: Rule) => exportRulesAction([rowItem.rule_id], dispatch), + enabled: (rowItem: Rule) => !rowItem.immutable, }, { description: i18n.DELETE_RULE, icon: 'trash', name: i18n.DELETE_RULE, - onClick: (rowItem: TableData) => deleteRulesAction([rowItem.id], dispatch, dispatchToaster), + onClick: (rowItem: Rule) => { + deleteRulesAction([rowItem.id], dispatch, dispatchToaster); + reFetchRules(true); + }, }, ]; -type RulesColumns = EuiBasicTableColumn | EuiTableActionsColumnType; +type RulesColumns = EuiBasicTableColumn | EuiTableActionsColumnType; + +interface GetColumns { + dispatch: React.Dispatch; + dispatchToaster: Dispatch; + history: H.History; + hasNoPermissions: boolean; + loadingRuleIds: string[]; + reFetchRules: (refreshPrePackagedRule?: boolean) => void; +} // Michael: Are we able to do custom, in-table-header filters, as shown in my wireframes? -export const getColumns = ( - dispatch: React.Dispatch, - dispatchToaster: Dispatch, - history: H.History, - hasNoPermissions: boolean -): RulesColumns[] => { +export const getColumns = ({ + dispatch, + dispatchToaster, + history, + hasNoPermissions, + loadingRuleIds, + reFetchRules, +}: GetColumns): RulesColumns[] => { const cols: RulesColumns[] = [ { - field: 'rule', + field: 'name', name: i18n.COLUMN_RULE, - render: (value: TableData['rule']) => {value.name}, + render: (value: Rule['name'], item: Rule) => ( + {value} + ), truncateText: true, width: '24%', }, @@ -93,14 +113,14 @@ export const getColumns = ( { field: 'severity', name: i18n.COLUMN_SEVERITY, - render: (value: TableData['severity']) => , + render: (value: Rule['severity']) => , truncateText: true, width: '16%', }, { - field: 'statusDate', + field: 'status_date', name: i18n.COLUMN_LAST_COMPLETE_RUN, - render: (value: TableData['statusDate']) => { + render: (value: Rule['status_date']) => { return value == null ? ( getEmptyTagValue() ) : ( @@ -114,7 +134,7 @@ export const getColumns = ( { field: 'status', name: i18n.COLUMN_LAST_RESPONSE, - render: (value: TableData['status']) => { + render: (value: Rule['status']) => { return ( <> @@ -129,7 +149,7 @@ export const getColumns = ( { field: 'tags', name: i18n.COLUMN_TAGS, - render: (value: TableData['tags']) => ( + render: (value: Rule['tags']) => ( {value.map((tag, i) => ( @@ -145,13 +165,13 @@ export const getColumns = ( align: 'center', field: 'activate', name: i18n.COLUMN_ACTIVATE, - render: (value: TableData['activate'], item: TableData) => ( + render: (value: Rule['enabled'], item: Rule) => ( ), sortable: true, @@ -160,9 +180,9 @@ export const getColumns = ( ]; const actions: RulesColumns[] = [ { - actions: getActions(dispatch, dispatchToaster, history), + actions: getActions(dispatch, dispatchToaster, history, reFetchRules), width: '40px', - } as EuiTableActionsColumnType, + } as EuiTableActionsColumnType, ]; return hasNoPermissions ? cols : [...cols, ...actions]; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/helpers.test.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/helpers.test.tsx index e925161444e42..c60933733587d 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/helpers.test.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/helpers.test.tsx @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { bucketRulesResponse, formatRules } from './helpers'; -import { mockRule, mockRuleError, mockRules, mockTableData } from './__mocks__/mock'; +import { bucketRulesResponse } from './helpers'; +import { mockRule, mockRuleError } from './__mocks__/mock'; import uuid from 'uuid'; import { Rule, RuleError } from '../../../../containers/detection_engine/rules'; @@ -15,20 +15,6 @@ describe('AllRulesTable Helpers', () => { const mockRuleError1: Readonly = mockRuleError(uuid.v4()); const mockRuleError2: Readonly = mockRuleError(uuid.v4()); - describe('formatRules', () => { - test('formats rules with no selection', () => { - const formattedRules = formatRules(mockRules); - expect(formattedRules).toEqual(mockTableData); - }); - - test('formats rules with selection', () => { - const mockTableDataWithSelected = [...mockTableData]; - mockTableDataWithSelected[0].isLoading = true; - const formattedRules = formatRules(mockRules, [mockRules[0].id]); - expect(formattedRules).toEqual(mockTableDataWithSelected); - }); - }); - describe('bucketRulesResponse', () => { test('buckets empty response', () => { const bucketedResponse = bucketRulesResponse([]); diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/helpers.ts index 9a523536694d9..5ce26144a4d9c 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/helpers.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/helpers.ts @@ -9,32 +9,6 @@ import { RuleError, RuleResponseBuckets, } from '../../../../containers/detection_engine/rules'; -import { TableData } from '../types'; - -/** - * Formats rules into the correct format for the AllRulesTable - * - * @param rules as returned from the Rules API - * @param selectedIds ids of the currently selected rules - */ -export const formatRules = (rules: Rule[], selectedIds?: string[]): TableData[] => - rules.map(rule => ({ - id: rule.id, - immutable: rule.immutable, - rule_id: rule.rule_id, - rule: { - href: `#/detections/rules/id/${encodeURIComponent(rule.id)}`, - name: rule.name, - }, - risk_score: rule.risk_score, - severity: rule.severity, - tags: rule.tags ?? [], - activate: rule.enabled, - status: rule.status ?? null, - statusDate: rule.status_date ?? null, - sourceRule: rule, - isLoading: selectedIds?.includes(rule.id) ?? false, - })); /** * Separates rules/errors from bulk rules API response (create/update/delete) @@ -52,14 +26,11 @@ export const bucketRulesResponse = (response: Array) => ); export const showRulesTable = ({ - isInitialLoad, rulesCustomInstalled, rulesInstalled, }: { - isInitialLoad: boolean; rulesCustomInstalled: number | null; rulesInstalled: number | null; }) => - !isInitialLoad && - ((rulesCustomInstalled != null && rulesCustomInstalled > 0) || - (rulesInstalled != null && rulesInstalled > 0)); + (rulesCustomInstalled != null && rulesCustomInstalled > 0) || + (rulesInstalled != null && rulesInstalled > 0); diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/index.tsx index b304d77f2e276..79fec526faf48 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/index.tsx @@ -11,15 +11,16 @@ import { EuiLoadingContent, EuiSpacer, } from '@elastic/eui'; -import { isEmpty } from 'lodash/fp'; -import React, { useCallback, useEffect, useMemo, useReducer, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useReducer, useRef, useState } from 'react'; import { useHistory } from 'react-router-dom'; +import styled from 'styled-components'; import uuid from 'uuid'; import { useRules, CreatePreBuiltRules, FilterOptions, + Rule, } from '../../../../containers/detection_engine/rules'; import { HeaderSection } from '../../../../components/header_section'; import { @@ -36,35 +37,39 @@ import { PrePackagedRulesPrompt } from '../components/pre_packaged_rules/load_em import { RuleDownloader } from '../components/rule_downloader'; import { getPrePackagedRuleStatus } from '../helpers'; import * as i18n from '../translations'; -import { EuiBasicTableOnChange, TableData } from '../types'; +import { EuiBasicTableOnChange } from '../types'; import { getBatchItems } from './batch_actions'; import { getColumns } from './columns'; import { showRulesTable } from './helpers'; import { allRulesReducer, State } from './reducer'; import { RulesTableFilters } from './rules_table_filters/rules_table_filters'; +// EuiBasicTable give me a hardtime with adding the ref attributes so I went the easy way +// after few hours of fight with typescript !!!! I lost :( +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const MyEuiBasicTable = styled(EuiBasicTable as any)`` as any; + const initialState: State = { - isLoading: true, - rules: [], - tableData: [], - selectedItems: [], - refreshToggle: true, - pagination: { - page: 1, - perPage: 20, - total: 0, - }, + exportRuleIds: [], filterOptions: { filter: '', sortField: 'enabled', sortOrder: 'desc', }, + loadingRuleIds: [], + loadingRulesAction: null, + pagination: { + page: 1, + perPage: 20, + total: 0, + }, + rules: [], + selectedRuleIds: [], }; interface AllRulesProps { createPrePackagedRules: CreatePreBuiltRules | null; hasNoPermissions: boolean; - importCompleteToggle: boolean; loading: boolean; loadingCreatePrePackagedRules: boolean; refetchPrePackagedRulesStatus: () => void; @@ -72,7 +77,7 @@ interface AllRulesProps { rulesInstalled: number | null; rulesNotInstalled: number | null; rulesNotUpdated: number | null; - setRefreshRulesData: (refreshRule: () => void) => void; + setRefreshRulesData: (refreshRule: (refreshPrePackagedRule?: boolean) => void) => void; } /** @@ -87,7 +92,6 @@ export const AllRules = React.memo( ({ createPrePackagedRules, hasNoPermissions, - importCompleteToggle, loading, loadingCreatePrePackagedRules, refetchPrePackagedRulesStatus, @@ -97,24 +101,36 @@ export const AllRules = React.memo( rulesNotUpdated, setRefreshRulesData, }) => { + const [initLoading, setInitLoading] = useState(true); + const tableRef = useRef(); const [ { - exportPayload, + exportRuleIds, filterOptions, - isLoading, - refreshToggle, - selectedItems, - tableData, + loadingRuleIds, + loadingRulesAction, pagination, + rules, + selectedRuleIds, }, dispatch, - ] = useReducer(allRulesReducer, initialState); + ] = useReducer(allRulesReducer(tableRef), initialState); const history = useHistory(); - const [oldRefreshToggle, setOldRefreshToggle] = useState(refreshToggle); - const [isInitialLoad, setIsInitialLoad] = useState(true); - const [isGlobalLoading, setIsGlobalLoad] = useState(false); const [, dispatchToaster] = useStateToaster(); - const [isLoadingRules, rulesData, reFetchRulesData] = useRules(pagination, filterOptions); + + const setRules = useCallback((newRules: Rule[]) => { + dispatch({ + type: 'setRules', + rules: newRules, + }); + }, []); + + const [isLoadingRules, , reFetchRulesData] = useRules({ + pagination, + filterOptions, + refetchPrePackagedRulesStatus, + dispatchRulesInReducer: setRules, + }); const prePackagedRuleStatus = getPrePackagedRuleStatus( rulesInstalled, @@ -125,10 +141,18 @@ export const AllRules = React.memo( const getBatchItemsPopoverContent = useCallback( (closePopover: () => void) => ( ), - [selectedItems, dispatch, dispatchToaster, history] + [dispatch, dispatchToaster, loadingRuleIds, reFetchRulesData, rules, selectedRuleIds] ); const tableOnChangeCallback = useCallback( @@ -146,46 +170,19 @@ export const AllRules = React.memo( ); const columns = useMemo(() => { - return getColumns(dispatch, dispatchToaster, history, hasNoPermissions); - }, [dispatch, dispatchToaster, history]); - - useEffect(() => { - dispatch({ type: 'loading', isLoading: isLoadingRules }); - }, [isLoadingRules]); - - useEffect(() => { - if (!isLoadingRules && !loading && isInitialLoad) { - setIsInitialLoad(false); - } - }, [isInitialLoad, isLoadingRules, loading]); - - useEffect(() => { - if (!isGlobalLoading && (isLoadingRules || isLoading)) { - setIsGlobalLoad(true); - } else if (isGlobalLoading && !isLoadingRules && !isLoading) { - setIsGlobalLoad(false); - } - }, [setIsGlobalLoad, isGlobalLoading, isLoadingRules, isLoading]); - - useEffect(() => { - if (!isInitialLoad) { - dispatch({ type: 'refresh' }); - } - }, [importCompleteToggle]); - - useEffect(() => { - if (!isInitialLoad && reFetchRulesData != null && oldRefreshToggle !== refreshToggle) { - setOldRefreshToggle(refreshToggle); - reFetchRulesData(); - refetchPrePackagedRulesStatus(); - } - }, [ - isInitialLoad, - refreshToggle, - oldRefreshToggle, - reFetchRulesData, - refetchPrePackagedRulesStatus, - ]); + return getColumns({ + dispatch, + dispatchToaster, + history, + hasNoPermissions, + loadingRuleIds: + loadingRulesAction != null && + (loadingRulesAction === 'enable' || loadingRulesAction === 'disable') + ? loadingRuleIds + : [], + reFetchRules: reFetchRulesData, + }); + }, [dispatch, dispatchToaster, history, loadingRuleIds, loadingRulesAction, reFetchRulesData]); useEffect(() => { if (reFetchRulesData != null) { @@ -194,31 +191,25 @@ export const AllRules = React.memo( }, [reFetchRulesData, setRefreshRulesData]); useEffect(() => { - dispatch({ - type: 'updateRules', - rules: rulesData.data, - pagination: { - page: rulesData.page, - perPage: rulesData.perPage, - total: rulesData.total, - }, - }); - }, [rulesData]); + if (initLoading && !loading && !isLoadingRules) { + setInitLoading(false); + } + }, [initLoading, loading, isLoadingRules]); const handleCreatePrePackagedRules = useCallback(async () => { - if (createPrePackagedRules != null) { + if (createPrePackagedRules != null && reFetchRulesData != null) { await createPrePackagedRules(); - dispatch({ type: 'refresh' }); + reFetchRulesData(true); } - }, [createPrePackagedRules]); + }, [createPrePackagedRules, reFetchRulesData]); const euiBasicTableSelectionProps = useMemo( () => ({ - selectable: (item: TableData) => !item.isLoading, - onSelectionChange: (selected: TableData[]) => - dispatch({ type: 'setSelected', selectedItems: selected }), + selectable: (item: Rule) => !loadingRuleIds.includes(item.id), + onSelectionChange: (selected: Rule[]) => + dispatch({ type: 'selectedRuleIds', ids: selected.map(r => r.id) }), }), - [] + [loadingRuleIds] ); const onFilterChangedCallback = useCallback((newFilterOptions: Partial) => { @@ -237,12 +228,25 @@ export const AllRules = React.memo( ); }, []); + const isLoadingAnActionOnRule = useMemo(() => { + if ( + loadingRuleIds.length > 0 && + (loadingRulesAction === 'disable' || loadingRulesAction === 'enable') + ) { + return false; + } else if (loadingRuleIds.length > 0) { + return true; + } + return false; + }, [loadingRuleIds, loadingRulesAction]); + return ( <> { + dispatch({ type: 'loadingRuleIds', ids: [], actionType: null }); dispatchToaster({ type: 'addToaster', toast: { @@ -256,22 +260,17 @@ export const AllRules = React.memo( /> - + <> - {((rulesCustomInstalled && rulesCustomInstalled > 0) || - (rulesInstalled != null && rulesInstalled > 0)) && ( - - - - )} - {isInitialLoad && ( - - )} - {isGlobalLoading && !isEmpty(tableData) && !isInitialLoad && ( + + + + + {(loading || isLoadingRules || isLoadingAnActionOnRule) && !initLoading && ( )} {rulesCustomInstalled != null && @@ -283,7 +282,10 @@ export const AllRules = React.memo( userHasNoPermissions={hasNoPermissions} /> )} - {showRulesTable({ isInitialLoad, rulesCustomInstalled, rulesInstalled }) && ( + {initLoading && ( + + )} + {showRulesTable({ rulesCustomInstalled, rulesInstalled }) && !initLoading && ( <> @@ -292,7 +294,7 @@ export const AllRules = React.memo( - {i18n.SELECTED_RULES(selectedItems.length)} + {i18n.SELECTED_RULES(selectedRuleIds.length)} {!hasNoPermissions && ( ( )} dispatch({ type: 'refresh' })} + onClick={() => reFetchRulesData(true)} > {i18n.REFRESH} - - ( totalItemCount: pagination.total, pageSizeOptions: [5, 10, 20, 50, 100, 200, 300], }} - sorting={{ sort: { field: 'activate', direction: filterOptions.sortOrder } }} + ref={tableRef} + sorting={{ sort: { field: 'enabled', direction: filterOptions.sortOrder } }} selection={hasNoPermissions ? undefined : euiBasicTableSelectionProps} /> diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/reducer.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/reducer.ts index 3634a16cbf6ac..54da43efd66d9 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/reducer.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/reducer.ts @@ -4,34 +4,30 @@ * you may not use this file except in compliance with the Elastic License. */ +import { EuiBasicTable } from '@elastic/eui'; import { FilterOptions, PaginationOptions, Rule, } from '../../../../containers/detection_engine/rules'; -import { TableData } from '../types'; -import { formatRules } from './helpers'; +type LoadingRuleAction = 'duplicate' | 'enable' | 'disable' | 'export' | 'delete' | null; export interface State { - isLoading: boolean; - rules: Rule[]; - selectedItems: TableData[]; - pagination: PaginationOptions; + exportRuleIds: string[]; filterOptions: FilterOptions; - refreshToggle: boolean; - tableData: TableData[]; - exportPayload?: Rule[]; + loadingRuleIds: string[]; + loadingRulesAction: LoadingRuleAction; + pagination: PaginationOptions; + rules: Rule[]; + selectedRuleIds: string[]; } export type Action = - | { type: 'refresh' } - | { type: 'loading'; isLoading: boolean } - | { type: 'deleteRules'; rules: Rule[] } - | { type: 'duplicate'; rule: Rule } - | { type: 'setExportPayload'; exportPayload?: Rule[] } - | { type: 'setSelected'; selectedItems: TableData[] } - | { type: 'updateLoading'; ids: string[]; isLoading: boolean } - | { type: 'updateRules'; rules: Rule[]; pagination?: PaginationOptions } + | { type: 'exportRuleIds'; ids: string[] } + | { type: 'loadingRuleIds'; ids: string[]; actionType: LoadingRuleAction } + | { type: 'selectedRuleIds'; ids: string[] } + | { type: 'setRules'; rules: Rule[] } + | { type: 'updateRules'; rules: Rule[] } | { type: 'updatePagination'; pagination: Partial } | { type: 'updateFilterOptions'; @@ -40,54 +36,71 @@ export type Action = } | { type: 'failure' }; -export const allRulesReducer = (state: State, action: Action): State => { +export const allRulesReducer = ( + tableRef: React.MutableRefObject | undefined> +) => (state: State, action: Action): State => { switch (action.type) { - case 'refresh': { + case 'exportRuleIds': { return { ...state, - refreshToggle: !state.refreshToggle, + loadingRuleIds: action.ids, + loadingRulesAction: 'export', + exportRuleIds: action.ids, }; } - case 'updateRules': { - // If pagination included, this was a hard refresh - if (action.pagination) { - return { - ...state, - rules: action.rules, - pagination: action.pagination, - tableData: formatRules(action.rules), - }; + case 'loadingRuleIds': { + return { + ...state, + loadingRuleIds: action.actionType == null ? [] : [...state.loadingRuleIds, ...action.ids], + loadingRulesAction: action.actionType, + }; + } + case 'selectedRuleIds': { + return { + ...state, + selectedRuleIds: action.ids, + }; + } + case 'setRules': { + if ( + tableRef != null && + tableRef.current != null && + tableRef.current.changeSelection != null + ) { + tableRef.current.changeSelection([]); } - const ruleIds = state.rules.map(r => r.rule_id); - const updatedRules = action.rules.reverse().reduce((rules, updatedRule) => { - let newRules = rules; - if (ruleIds.includes(updatedRule.rule_id)) { - newRules = newRules.map(r => (updatedRule.rule_id === r.rule_id ? updatedRule : r)); - } else { - newRules = [...newRules, updatedRule]; - } - return newRules; - }, state.rules); - - // Update enabled on selectedItems so that batch actions show correct available actions - const updatedRuleIdToState = action.rules.reduce>( - (acc, r) => ({ ...acc, [r.id]: r.enabled }), - {} - ); - const updatedSelectedItems = state.selectedItems.map(selectedItem => - Object.keys(updatedRuleIdToState).includes(selectedItem.id) - ? { ...selectedItem, activate: updatedRuleIdToState[selectedItem.id] } - : selectedItem - ); - return { ...state, - rules: updatedRules, - tableData: formatRules(updatedRules), - selectedItems: updatedSelectedItems, + rules: action.rules, + selectedRuleIds: [], + loadingRuleIds: [], + loadingRulesAction: null, }; } + case 'updateRules': { + if (state.rules != null) { + const ruleIds = state.rules.map(r => r.id); + const updatedRules = action.rules.reduce((rules, updatedRule) => { + let newRules = rules; + if (ruleIds.includes(updatedRule.id)) { + newRules = newRules.map(r => (updatedRule.id === r.id ? updatedRule : r)); + } else { + newRules = [...newRules, updatedRule]; + } + return newRules; + }, state.rules); + const updatedRuleIds = action.rules.map(r => r.id); + const newLoadingRuleIds = state.loadingRuleIds.filter(id => !updatedRuleIds.includes(id)); + return { + ...state, + rules: updatedRules, + loadingRuleIds: newLoadingRuleIds, + loadingRulesAction: newLoadingRuleIds.length === 0 ? null : state.loadingRulesAction, + }; + } + return state; + } case 'updatePagination': { return { ...state, @@ -110,51 +123,12 @@ export const allRulesReducer = (state: State, action: Action): State => { }, }; } - case 'deleteRules': { - const deletedRuleIds = action.rules.map(r => r.rule_id); - const updatedRules = state.rules.reduce( - (rules, rule) => (deletedRuleIds.includes(rule.rule_id) ? rules : [...rules, rule]), - [] - ); - return { - ...state, - rules: updatedRules, - tableData: formatRules(updatedRules), - refreshToggle: !state.refreshToggle, - }; - } - case 'setSelected': { - return { - ...state, - selectedItems: action.selectedItems, - }; - } - case 'updateLoading': { - return { - ...state, - rules: state.rules, - tableData: formatRules(state.rules, action.ids), - }; - } - case 'loading': { - return { - ...state, - isLoading: action.isLoading, - }; - } case 'failure': { return { ...state, - isLoading: false, rules: [], }; } - case 'setExportPayload': { - return { - ...state, - exportPayload: [...(action.exportPayload ?? [])], - }; - } default: return state; } diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/rules_table_filters/rules_table_filters.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/rules_table_filters/rules_table_filters.tsx index fa4f6a874ca5e..ddb8894c206b5 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/rules_table_filters/rules_table_filters.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/rules_table_filters/rules_table_filters.tsx @@ -41,7 +41,11 @@ const RulesTableFiltersComponent = ({ const [selectedTags, setSelectedTags] = useState([]); const [showCustomRules, setShowCustomRules] = useState(false); const [showElasticRules, setShowElasticRules] = useState(false); - const [isLoadingTags, tags] = useTags(); + const [isLoadingTags, tags, reFetchTags] = useTags(); + + useEffect(() => { + reFetchTags(); + }, [rulesCustomInstalled, rulesInstalled]); // Propagate filter changes to parent useEffect(() => { diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_actions_overflow/__snapshots__/index.test.tsx.snap b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_actions_overflow/__snapshots__/index.test.tsx.snap index 9bd2fab23ac99..9355d0ae2cccb 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_actions_overflow/__snapshots__/index.test.tsx.snap +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_actions_overflow/__snapshots__/index.test.tsx.snap @@ -58,6 +58,7 @@ exports[`RuleActionsOverflow renders correctly against snapshot 1`] = ` `; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_actions_overflow/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_actions_overflow/index.tsx index b52c10881cf42..7c8926c2064c7 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_actions_overflow/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_actions_overflow/index.tsx @@ -48,7 +48,7 @@ const RuleActionsOverflowComponent = ({ userHasNoPermissions, }: RuleActionsOverflowComponentProps) => { const [isPopoverOpen, setIsPopoverOpen] = useState(false); - const [rulesToExport, setRulesToExport] = useState(undefined); + const [rulesToExport, setRulesToExport] = useState([]); const history = useHistory(); const [, dispatchToaster] = useStateToaster(); @@ -66,7 +66,7 @@ const RuleActionsOverflowComponent = ({ disabled={userHasNoPermissions} onClick={async () => { setIsPopoverOpen(false); - await duplicateRulesAction([rule], noop, dispatchToaster); + await duplicateRulesAction([rule], [rule.id], noop, dispatchToaster); }} > {i18nActions.DUPLICATE_RULE} @@ -75,9 +75,9 @@ const RuleActionsOverflowComponent = ({ key={i18nActions.EXPORT_RULE} icon="indexEdit" disabled={userHasNoPermissions || rule.immutable} - onClick={async () => { + onClick={() => { setIsPopoverOpen(false); - setRulesToExport([rule]); + setRulesToExport([rule.id]); }} > {i18nActions.EXPORT_RULE} @@ -131,7 +131,7 @@ const RuleActionsOverflowComponent = ({ { displaySuccessToast( i18nActions.SUCCESSFULLY_EXPORTED_RULES(exportCount), diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_downloader/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_downloader/index.tsx index b41265adea6b1..5d3086051a6e2 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_downloader/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/rule_downloader/index.tsx @@ -7,7 +7,7 @@ import React, { useEffect, useRef } from 'react'; import styled from 'styled-components'; import { isFunction } from 'lodash/fp'; -import { exportRules, Rule } from '../../../../../containers/detection_engine/rules'; +import { exportRules } from '../../../../../containers/detection_engine/rules'; import { displayErrorToast, useStateToaster } from '../../../../../components/toasters'; import * as i18n from './translations'; @@ -17,7 +17,7 @@ const InvisibleAnchor = styled.a` export interface RuleDownloaderProps { filename: string; - rules?: Rule[]; + ruleIds?: string[]; onExportComplete: (exportCount: number) => void; } @@ -30,7 +30,7 @@ export interface RuleDownloaderProps { */ export const RuleDownloaderComponent = ({ filename, - rules, + ruleIds, onExportComplete, }: RuleDownloaderProps) => { const anchorRef = useRef(null); @@ -41,10 +41,10 @@ export const RuleDownloaderComponent = ({ const abortCtrl = new AbortController(); async function exportData() { - if (anchorRef && anchorRef.current && rules != null) { + if (anchorRef && anchorRef.current && ruleIds != null && ruleIds.length > 0) { try { const exportResponse = await exportRules({ - ruleIds: rules.map(r => r.rule_id), + ruleIds, signal: abortCtrl.signal, }); @@ -61,7 +61,7 @@ export const RuleDownloaderComponent = ({ window.URL.revokeObjectURL(objectURL); } - onExportComplete(rules.length); + onExportComplete(ruleIds.length); } } catch (error) { if (isSubscribed) { @@ -77,7 +77,7 @@ export const RuleDownloaderComponent = ({ isSubscribed = false; abortCtrl.abort(); }; - }, [rules]); + }, [ruleIds]); return ; }; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/index.tsx index 0c53ad19a3574..20185c2eda816 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/index.tsx @@ -26,11 +26,10 @@ import { UpdatePrePackagedRulesCallOut } from './components/pre_packaged_rules/u import { getPrePackagedRuleStatus, redirectToDetections } from './helpers'; import * as i18n from './translations'; -type Func = () => void; +type Func = (refreshPrePackagedRule?: boolean) => void; const RulesPageComponent: React.FC = () => { const [showImportModal, setShowImportModal] = useState(false); - const [importCompleteToggle, setImportCompleteToggle] = useState(false); const refreshRulesData = useRef(null); const { loading, @@ -67,14 +66,18 @@ const RulesPageComponent: React.FC = () => { const userHasNoPermissions = canUserCRUD != null && hasManageApiKey != null ? !canUserCRUD || !hasManageApiKey : false; + const handleRefreshRules = useCallback(async () => { + if (refreshRulesData.current != null) { + refreshRulesData.current(true); + } + }, [refreshRulesData]); + const handleCreatePrePackagedRules = useCallback(async () => { if (createPrePackagedRules != null) { await createPrePackagedRules(); - if (refreshRulesData.current != null) { - refreshRulesData.current(); - } + handleRefreshRules(); } - }, [createPrePackagedRules, refreshRulesData]); + }, [createPrePackagedRules, handleRefreshRules]); const handleRefetchPrePackagedRulesStatus = useCallback(() => { if (refetchPrePackagedRulesStatus != null) { @@ -96,7 +99,7 @@ const RulesPageComponent: React.FC = () => { setShowImportModal(false)} - importComplete={() => setImportCompleteToggle(!importCompleteToggle)} + importComplete={handleRefreshRules} /> { loading={loading || prePackagedRuleLoading} loadingCreatePrePackagedRules={loadingCreatePrePackagedRules} hasNoPermissions={userHasNoPermissions} - importCompleteToggle={importCompleteToggle} refetchPrePackagedRulesStatus={handleRefetchPrePackagedRulesStatus} rulesCustomInstalled={rulesCustomInstalled} rulesInstalled={rulesInstalled} diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts index 55eb45fb5ed9d..b2650dcc2b77e 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts @@ -5,7 +5,6 @@ */ import { Filter } from '../../../../../../../../src/plugins/data/common'; -import { Rule } from '../../../containers/detection_engine/rules'; import { FieldValueQueryBar } from './components/query_bar'; import { FormData, FormHook } from '../../shared_imports'; import { FieldValueTimeline } from './components/pick_timeline'; @@ -23,24 +22,6 @@ export interface EuiBasicTableOnChange { sort?: EuiBasicTableSortTypes; } -export interface TableData { - id: string; - immutable: boolean; - rule_id: string; - rule: { - href: string; - name: string; - }; - risk_score: number; - severity: string; - tags: string[]; - activate: boolean; - isLoading: boolean; - sourceRule: Rule; - status?: string | null; - statusDate?: string | null; -} - export enum RuleStep { defineRule = 'define-rule', aboutRule = 'about-rule', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index 51b7b132fc794..08a0589389966 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -5,7 +5,6 @@ */ import Hapi from 'hapi'; -import { countBy } from 'lodash/fp'; import uuid from 'uuid'; import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants'; @@ -45,8 +44,7 @@ export const createCreateRulesBulkRoute = ( } const ruleDefinitions = request.payload; - const mappedDuplicates = countBy('rule_id', ruleDefinitions); - const dupes = getDuplicates(mappedDuplicates); + const dupes = getDuplicates(ruleDefinitions, 'rule_id'); const rules = await Promise.all( ruleDefinitions diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts index b5b687c284b25..5fac3f79f359c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts @@ -20,7 +20,7 @@ import { } from './utils'; import { getResult } from '../__mocks__/request_responses'; import { INTERNAL_IDENTIFIER } from '../../../../../common/constants'; -import { OutputRuleAlertRest, ImportRuleAlertRest } from '../../types'; +import { OutputRuleAlertRest, ImportRuleAlertRest, RuleAlertParamsRest } from '../../types'; import { BulkError, ImportSuccessError } from '../utils'; import { sampleRule } from '../../signals/__mocks__/es_results'; import { getSimpleRule } from '../__mocks__/utils'; @@ -1222,20 +1222,32 @@ describe('utils', () => { describe('getDuplicates', () => { test("returns array of ruleIds showing the duplicate keys of 'value2' and 'value3'", () => { - const output = getDuplicates({ - value1: 1, - value2: 2, - value3: 2, - }); + const output = getDuplicates( + [ + { rule_id: 'value1' }, + { rule_id: 'value2' }, + { rule_id: 'value2' }, + { rule_id: 'value3' }, + { rule_id: 'value3' }, + {}, + {}, + ] as RuleAlertParamsRest[], + 'rule_id' + ); const expected = ['value2', 'value3']; expect(output).toEqual(expected); }); test('returns null when given a map of no duplicates', () => { - const output = getDuplicates({ - value1: 1, - value2: 1, - value3: 1, - }); + const output = getDuplicates( + [ + { rule_id: 'value1' }, + { rule_id: 'value2' }, + { rule_id: 'value3' }, + {}, + {}, + ] as RuleAlertParamsRest[], + 'rule_id' + ); const expected: string[] = []; expect(output).toEqual(expected); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts index ab80e1570c31c..7004bf2088ef2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts @@ -4,8 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { pickBy } from 'lodash/fp'; -import { Dictionary } from 'lodash'; +import { pickBy, countBy } from 'lodash/fp'; import { SavedObject } from 'kibana/server'; import uuid from 'uuid'; import { INTERNAL_IDENTIFIER } from '../../../../../common/constants'; @@ -18,7 +17,7 @@ import { isRuleStatusFindTypes, isRuleStatusSavedObjectType, } from '../../rules/types'; -import { OutputRuleAlertRest, ImportRuleAlertRest } from '../../types'; +import { OutputRuleAlertRest, ImportRuleAlertRest, RuleAlertParamsRest } from '../../types'; import { createBulkErrorObject, BulkError, @@ -224,10 +223,14 @@ export const transformOrImportError = ( } }; -export const getDuplicates = (lodashDict: Dictionary): string[] => { - const hasDuplicates = Object.values(lodashDict).some(i => i > 1); +export const getDuplicates = (ruleDefinitions: RuleAlertParamsRest[], by: 'rule_id'): string[] => { + const mappedDuplicates = countBy( + by, + ruleDefinitions.filter(r => r[by] != null) + ); + const hasDuplicates = Object.values(mappedDuplicates).some(i => i > 1); if (hasDuplicates) { - return Object.keys(lodashDict).filter(key => lodashDict[key] > 1); + return Object.keys(mappedDuplicates).filter(key => mappedDuplicates[key] > 1); } return []; }; diff --git a/x-pack/plugins/actions/server/builtin_action_types/server_log.ts b/x-pack/plugins/actions/server/builtin_action_types/server_log.ts index 62406cfaf66e1..01355f2a34f92 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/server_log.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/server_log.ts @@ -12,8 +12,6 @@ import { Logger } from '../../../../../src/core/server'; import { ActionType, ActionTypeExecutorOptions, ActionTypeExecutorResult } from '../types'; import { withoutControlCharacters } from './lib/string_utils'; -const ACTION_NAME = 'Server log'; - // params definition export type ActionParamsType = TypeOf; @@ -37,7 +35,9 @@ const ParamsSchema = schema.object({ export function getActionType({ logger }: { logger: Logger }): ActionType { return { id: '.server-log', - name: ACTION_NAME, + name: i18n.translate('xpack.actions.builtin.serverLogTitle', { + defaultMessage: 'Server log', + }), validate: { params: ParamsSchema, }, @@ -56,7 +56,7 @@ async function executor( const sanitizedMessage = withoutControlCharacters(params.message); try { - logger[params.level](`${ACTION_NAME}: ${sanitizedMessage}`); + logger[params.level](`Server log: ${sanitizedMessage}`); } catch (err) { const message = i18n.translate('xpack.actions.builtin.serverLog.errorLoggingErrorMessage', { defaultMessage: 'error logging message', diff --git a/x-pack/plugins/actions/server/builtin_action_types/servicenow.test.ts b/x-pack/plugins/actions/server/builtin_action_types/servicenow.test.ts index a6c43f48fa803..9ae96cb23a5c3 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/servicenow.test.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/servicenow.test.ts @@ -49,7 +49,7 @@ beforeAll(() => { describe('get()', () => { test('should return correct action type', () => { expect(actionType.id).toEqual(ACTION_TYPE_ID); - expect(actionType.name).toEqual('servicenow'); + expect(actionType.name).toEqual('ServiceNow'); }); }); diff --git a/x-pack/plugins/actions/server/builtin_action_types/servicenow.ts b/x-pack/plugins/actions/server/builtin_action_types/servicenow.ts index 2d5c18207def3..0ad435281eba4 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/servicenow.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/servicenow.ts @@ -89,7 +89,9 @@ export function getActionType({ }): ActionType { return { id: '.servicenow', - name: 'servicenow', + name: i18n.translate('xpack.actions.builtin.servicenowTitle', { + defaultMessage: 'ServiceNow', + }), validate: { config: schema.object(ConfigSchemaProps, { validate: curry(validateConfig)(configurationUtilities), diff --git a/x-pack/plugins/apm/common/service_map.ts b/x-pack/plugins/apm/common/service_map.ts index 528aec2f70ad9..548b29346e483 100644 --- a/x-pack/plugins/apm/common/service_map.ts +++ b/x-pack/plugins/apm/common/service_map.ts @@ -4,6 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ +import { i18n } from '@kbn/i18n'; +import { ILicense } from '../../licensing/public'; + export interface ServiceConnectionNode { 'service.name': string; 'service.environment': string | null; @@ -30,3 +33,18 @@ export interface ServiceNodeMetrics { avgRequestsPerMinute: number | null; avgErrorsPerMinute: number | null; } + +export function isValidPlatinumLicense(license: ILicense) { + return ( + license.isActive && + (license.type === 'platinum' || license.type === 'trial') + ); +} + +export const invalidLicenseMessage = i18n.translate( + 'xpack.apm.serviceMap.invalidLicenseMessage', + { + defaultMessage: + "In order to access Service Maps, you must be subscribed to an Elastic Platinum license. With it, you'll have the ability to visualize your entire application stack along with your APM data." + } +); diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index 42232a8b89605..96579377c95e8 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -5,6 +5,6 @@ "kibanaVersion": "kibana", "configPath": ["xpack", "apm"], "ui": false, - "requiredPlugins": ["apm_oss", "data", "home"], + "requiredPlugins": ["apm_oss", "data", "home", "licensing"], "optionalPlugins": ["cloud", "usageCollection"] } diff --git a/x-pack/plugins/apm/server/lib/helpers/es_client.ts b/x-pack/plugins/apm/server/lib/helpers/es_client.ts index 86eb1dba507f0..c22084dbb7168 100644 --- a/x-pack/plugins/apm/server/lib/helpers/es_client.ts +++ b/x-pack/plugins/apm/server/lib/helpers/es_client.ts @@ -12,8 +12,9 @@ import { IndicesCreateParams, DeleteDocumentResponse } from 'elasticsearch'; -import { cloneDeep, isString, merge, uniqueId } from 'lodash'; +import { cloneDeep, isString, merge } from 'lodash'; import { KibanaRequest } from 'src/core/server'; +import chalk from 'chalk'; import { ESSearchRequest, ESSearchResponse @@ -126,6 +127,10 @@ interface ClientCreateOptions { export type ESClient = ReturnType; +function formatObj(obj: Record) { + return JSON.stringify(obj, null, 2); +} + export function getESClient( context: APMRequestHandlerContext, request: KibanaRequest, @@ -136,25 +141,49 @@ export function getESClient( callAsInternalUser } = context.core.elasticsearch.dataClient; - const callMethod = clientAsInternalUser - ? callAsInternalUser - : callAsCurrentUser; + async function callEs(operationName: string, params: Record) { + const startTime = process.hrtime(); + + let res: any; + let esError = null; + try { + res = clientAsInternalUser + ? await callAsInternalUser(operationName, params) + : await callAsCurrentUser(operationName, params); + } catch (e) { + // catch error and throw after outputting debug info + esError = e; + } + + if (context.params.query._debug) { + const highlightColor = esError ? 'bgRed' : 'inverse'; + const diff = process.hrtime(startTime); + const duration = `${Math.round(diff[0] * 1000 + diff[1] / 1e6)}ms`; + const routeInfo = `${request.route.method.toUpperCase()} ${ + request.route.path + }`; - const debug = context.params.query._debug; + console.log( + chalk.bold[highlightColor](`=== Debug: ${routeInfo} (${duration}) ===`) + ); + + if (operationName === 'search') { + console.log(`GET ${params.index}/_${operationName}`); + console.log(formatObj(params.body)); + } else { + console.log(chalk.bold('ES operation:'), operationName); - function withTime( - fn: (log: typeof console.log) => Promise - ): Promise { - const log = console.log.bind(console, uniqueId()); - if (!debug) { - return fn(log); + console.log(chalk.bold('ES query:')); + console.log(formatObj(params)); + } + console.log(`\n`); } - const time = process.hrtime(); - return fn(log).then(data => { - const now = process.hrtime(time); - log(`took: ${Math.round(now[0] * 1000 + now[1] / 1e6)}ms`); - return data; - }); + + if (esError) { + throw esError; + } + + return res; } return { @@ -171,40 +200,25 @@ export function getESClient( apmOptions ); - return withTime(log => { - if (context.params.query._debug) { - log(`--DEBUG ES QUERY--`); - log( - `${request.url.pathname} ${JSON.stringify(context.params.query)}` - ); - log(`GET ${nextParams.index}/_search`); - log(JSON.stringify(nextParams.body, null, 2)); - } - - return (callMethod('search', nextParams) as unknown) as Promise< - ESSearchResponse - >; - }); + return callEs('search', nextParams); }, index: (params: APMIndexDocumentParams) => { - return withTime(() => callMethod('index', params)); + return callEs('index', params); }, delete: (params: IndicesDeleteParams): Promise => { - return withTime(() => callMethod('delete', params)); + return callEs('delete', params); }, indicesCreate: (params: IndicesCreateParams) => { - return withTime(() => callMethod('indices.create', params)); + return callEs('indices.create', params); }, hasPrivileges: ( params: IndexPrivilegesParams ): Promise => { - return withTime(() => - callMethod('transport.request', { - method: 'POST', - path: '/_security/user/_has_privileges', - body: params - }) - ); + return callEs('transport.request', { + method: 'POST', + path: '/_security/user/_has_privileges', + body: params + }); } }; } diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index de23c4c5dd383..adc80cb43620b 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -19,6 +19,7 @@ import { HomeServerPluginSetup } from '../../../../src/plugins/home/server'; import { tutorialProvider } from './tutorial'; import { CloudSetup } from '../../cloud/server'; import { getInternalSavedObjectsClient } from './lib/helpers/get_internal_saved_objects_client'; +import { LicensingPluginSetup } from '../../licensing/public'; export interface LegacySetup { server: Server; @@ -44,6 +45,7 @@ export class APMPlugin implements Plugin { plugins: { apm_oss: APMOSSPlugin extends Plugin ? TSetup : never; home: HomeServerPluginSetup; + licensing: LicensingPluginSetup; cloud?: CloudSetup; usageCollection?: UsageCollectionSetup; } diff --git a/x-pack/plugins/apm/server/routes/service_map.ts b/x-pack/plugins/apm/server/routes/service_map.ts index 584598805f8b3..bead0445d6ccc 100644 --- a/x-pack/plugins/apm/server/routes/service_map.ts +++ b/x-pack/plugins/apm/server/routes/service_map.ts @@ -4,13 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -import * as t from 'io-ts'; import Boom from 'boom'; +import * as t from 'io-ts'; +import { + invalidLicenseMessage, + isValidPlatinumLicense +} from '../../common/service_map'; import { setupRequest } from '../lib/helpers/setup_request'; -import { createRoute } from './create_route'; -import { uiFiltersRt, rangeRt } from './default_api_types'; import { getServiceMap } from '../lib/service_map/get_service_map'; import { getServiceMapServiceNodeInfo } from '../lib/service_map/get_service_map_service_node_info'; +import { createRoute } from './create_route'; +import { rangeRt, uiFiltersRt } from './default_api_types'; export const serviceMapRoute = createRoute(() => ({ path: '/api/apm/service-map', @@ -26,6 +30,10 @@ export const serviceMapRoute = createRoute(() => ({ if (!context.config['xpack.apm.serviceMapEnabled']) { throw Boom.notFound(); } + if (!isValidPlatinumLicense(context.licensing.license)) { + throw Boom.forbidden(invalidLicenseMessage); + } + const setup = await setupRequest(context, request); const { query: { serviceName, environment, after } @@ -51,6 +59,9 @@ export const serviceMapServiceNodeRoute = createRoute(() => ({ if (!context.config['xpack.apm.serviceMapEnabled']) { throw Boom.notFound(); } + if (!isValidPlatinumLicense(context.licensing.license)) { + throw Boom.forbidden(invalidLicenseMessage); + } const setup = await setupRequest(context, request); const { diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 4a627d48c3cf0..3c7d0ce47acb7 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -7263,7 +7263,6 @@ "xpack.maps.source.esGrid.finestDropdownOption": "最も細かい", "xpack.maps.source.esGrid.geospatialFieldLabel": "地理空間フィールド", "xpack.maps.source.esGrid.indexPatternLabel": "インデックスパターン", - "xpack.maps.source.esGrid.inspectorDescription": "Elasticsearch ジオグリッド集約リクエスト", "xpack.maps.source.esGrid.metricsLabel": "メトリック", "xpack.maps.source.esGrid.noIndexPatternErrorMessage": "インデックスパターン {id} が見つかりません", "xpack.maps.source.esGrid.resolutionParamErrorMessage": "グリッド解像度パラメーターが認識されません: {resolution}", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index e0ef4a7a1ebdb..b262be626aa53 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -7263,7 +7263,6 @@ "xpack.maps.source.esGrid.finestDropdownOption": "最精致化", "xpack.maps.source.esGrid.geospatialFieldLabel": "地理空间字段", "xpack.maps.source.esGrid.indexPatternLabel": "索引模式", - "xpack.maps.source.esGrid.inspectorDescription": "Elasticsearch 地理网格聚合请求", "xpack.maps.source.esGrid.metricsLabel": "指标", "xpack.maps.source.esGrid.noIndexPatternErrorMessage": "找不到索引模式 {id}", "xpack.maps.source.esGrid.resolutionParamErrorMessage": "无法识别网格分辨率参数：{resolution}", diff --git a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts index 87ae5231d1031..1dd069bb907d1 100644 --- a/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts +++ b/x-pack/test/functional/apps/discover/feature_controls/discover_security.ts @@ -28,8 +28,7 @@ export default function({ getPageObjects, getService }: FtrProviderContext) { await PageObjects.timePicker.setDefaultAbsoluteRange(); } - // FLAKY: https://github.com/elastic/kibana/issues/45348 - describe.skip('security', () => { + describe('security', () => { before(async () => { await esArchiver.load('discover/feature_controls/security'); await esArchiver.loadIfNeeded('logstash_functional'); diff --git a/x-pack/test/functional/es_archives/reporting/ecommerce/data.json.gz b/x-pack/test/functional/es_archives/reporting/ecommerce/data.json.gz index b38981c03417e..58ac5616651d4 100644 Binary files a/x-pack/test/functional/es_archives/reporting/ecommerce/data.json.gz and b/x-pack/test/functional/es_archives/reporting/ecommerce/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json.gz b/x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json.gz index c2050b3399ab7..2b204d0bde271 100644 Binary files a/x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json.gz and b/x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/reporting/nanos/data.json.gz b/x-pack/test/functional/es_archives/reporting/nanos/data.json.gz index d0531c7607736..2811c495aae2d 100644 Binary files a/x-pack/test/functional/es_archives/reporting/nanos/data.json.gz and b/x-pack/test/functional/es_archives/reporting/nanos/data.json.gz differ