From 517c34a7ebf6ad6e02dab08c04155f7e0ab17956 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Tue, 21 Jul 2020 11:34:04 +0200 Subject: [PATCH 001/104] preserve 401 errors from new es client (#71248) * intercept 401 error from new client in routing layer * improvements * lint * fix mocked client construction due to 7.9-rc1 bump * use default WWW-Authenticate value when not provided by ES 401 --- .../elasticsearch/client/errors.test.ts | 82 ++++++++++++ .../server/elasticsearch/client/errors.ts | 32 +++++ .../server/elasticsearch/client/mocks.test.ts | 6 + src/core/server/elasticsearch/client/mocks.ts | 15 ++- .../core_service.test.mocks.ts | 17 ++- .../integration_tests/core_services.test.ts | 117 +++++++++++++++++- src/core/server/http/router/router.ts | 34 ++++- 7 files changed, 288 insertions(+), 15 deletions(-) create mode 100644 src/core/server/elasticsearch/client/errors.test.ts create mode 100644 src/core/server/elasticsearch/client/errors.ts diff --git a/src/core/server/elasticsearch/client/errors.test.ts b/src/core/server/elasticsearch/client/errors.test.ts new file mode 100644 index 0000000000000..35ad4ca71f48c --- /dev/null +++ b/src/core/server/elasticsearch/client/errors.test.ts @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + ResponseError, + ConnectionError, + ConfigurationError, +} from '@elastic/elasticsearch/lib/errors'; +import { ApiResponse } from '@elastic/elasticsearch'; +import { isResponseError, isUnauthorizedError } from './errors'; + +const createApiResponseError = ({ + statusCode = 200, + headers = {}, + body = {}, +}: { + statusCode?: number; + headers?: Record; + body?: Record; +} = {}): ApiResponse => { + return { + body, + statusCode, + headers, + warnings: [], + meta: {} as any, + }; +}; + +describe('isResponseError', () => { + it('returns `true` when the input is a `ResponseError`', () => { + expect(isResponseError(new ResponseError(createApiResponseError()))).toBe(true); + }); + + it('returns `false` when the input is not a `ResponseError`', () => { + expect(isResponseError(new Error('foo'))).toBe(false); + expect(isResponseError(new ConnectionError('error', createApiResponseError()))).toBe(false); + expect(isResponseError(new ConfigurationError('foo'))).toBe(false); + }); +}); + +describe('isUnauthorizedError', () => { + it('returns true when the input is a `ResponseError` and statusCode === 401', () => { + expect( + isUnauthorizedError(new ResponseError(createApiResponseError({ statusCode: 401 }))) + ).toBe(true); + }); + + it('returns false when the input is a `ResponseError` and statusCode !== 401', () => { + expect( + isUnauthorizedError(new ResponseError(createApiResponseError({ statusCode: 200 }))) + ).toBe(false); + expect( + isUnauthorizedError(new ResponseError(createApiResponseError({ statusCode: 403 }))) + ).toBe(false); + expect( + isUnauthorizedError(new ResponseError(createApiResponseError({ statusCode: 500 }))) + ).toBe(false); + }); + + it('returns `false` when the input is not a `ResponseError`', () => { + expect(isUnauthorizedError(new Error('foo'))).toBe(false); + expect(isUnauthorizedError(new ConnectionError('error', createApiResponseError()))).toBe(false); + expect(isUnauthorizedError(new ConfigurationError('foo'))).toBe(false); + }); +}); diff --git a/src/core/server/elasticsearch/client/errors.ts b/src/core/server/elasticsearch/client/errors.ts new file mode 100644 index 0000000000000..31a27170e1155 --- /dev/null +++ b/src/core/server/elasticsearch/client/errors.ts @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; + +export type UnauthorizedError = ResponseError & { + statusCode: 401; +}; + +export function isResponseError(error: any): error is ResponseError { + return Boolean(error.body && error.statusCode && error.headers); +} + +export function isUnauthorizedError(error: any): error is UnauthorizedError { + return isResponseError(error) && error.statusCode === 401; +} diff --git a/src/core/server/elasticsearch/client/mocks.test.ts b/src/core/server/elasticsearch/client/mocks.test.ts index b882f8d0c5d79..a6ce95155331e 100644 --- a/src/core/server/elasticsearch/client/mocks.test.ts +++ b/src/core/server/elasticsearch/client/mocks.test.ts @@ -49,6 +49,12 @@ describe('Mocked client', () => { expectMocked(client.close); }); + it('used EventEmitter functions should be mocked', () => { + expectMocked(client.on); + expectMocked(client.off); + expectMocked(client.once); + }); + it('`child` should be mocked and return a mocked Client', () => { expectMocked(client.child); diff --git a/src/core/server/elasticsearch/client/mocks.ts b/src/core/server/elasticsearch/client/mocks.ts index 34e83922d4d86..ec2885dfdf922 100644 --- a/src/core/server/elasticsearch/client/mocks.ts +++ b/src/core/server/elasticsearch/client/mocks.ts @@ -54,13 +54,20 @@ const createInternalClientMock = (): DeeplyMockedKeys => { mockify(client, omittedProps); - client.transport = { + // client got some read-only (getter) properties + // so we need to extend it to override the getter-only props. + const mock: any = { ...client }; + + mock.transport = { request: jest.fn(), }; - client.close = jest.fn().mockReturnValue(Promise.resolve()); - client.child = jest.fn().mockImplementation(() => createInternalClientMock()); + mock.close = jest.fn().mockReturnValue(Promise.resolve()); + mock.child = jest.fn().mockImplementation(() => createInternalClientMock()); + mock.on = jest.fn(); + mock.off = jest.fn(); + mock.once = jest.fn(); - return (client as unknown) as DeeplyMockedKeys; + return (mock as unknown) as DeeplyMockedKeys; }; export type ElasticSearchClientMock = DeeplyMockedKeys; diff --git a/src/core/server/http/integration_tests/core_service.test.mocks.ts b/src/core/server/http/integration_tests/core_service.test.mocks.ts index c23724b7d332f..515dad5383c01 100644 --- a/src/core/server/http/integration_tests/core_service.test.mocks.ts +++ b/src/core/server/http/integration_tests/core_service.test.mocks.ts @@ -18,10 +18,12 @@ */ import { elasticsearchServiceMock } from '../../elasticsearch/elasticsearch_service.mock'; -export const clusterClientMock = jest.fn(); -export const clusterClientInstanceMock = elasticsearchServiceMock.createLegacyScopedClusterClient(); +export const MockLegacyScopedClusterClient = jest.fn(); +export const legacyClusterClientInstanceMock = elasticsearchServiceMock.createLegacyScopedClusterClient(); jest.doMock('../../elasticsearch/legacy/scoped_cluster_client', () => ({ - LegacyScopedClusterClient: clusterClientMock.mockImplementation(() => clusterClientInstanceMock), + LegacyScopedClusterClient: MockLegacyScopedClusterClient.mockImplementation( + () => legacyClusterClientInstanceMock + ), })); jest.doMock('elasticsearch', () => { @@ -34,3 +36,12 @@ jest.doMock('elasticsearch', () => { }, }; }); + +export const MockElasticsearchClient = jest.fn(); +jest.doMock('@elastic/elasticsearch', () => { + const real = jest.requireActual('@elastic/elasticsearch'); + return { + ...real, + Client: MockElasticsearchClient, + }; +}); diff --git a/src/core/server/http/integration_tests/core_services.test.ts b/src/core/server/http/integration_tests/core_services.test.ts index 3c5f22500e5e0..6338326626d54 100644 --- a/src/core/server/http/integration_tests/core_services.test.ts +++ b/src/core/server/http/integration_tests/core_services.test.ts @@ -17,14 +17,21 @@ * under the License. */ -import { clusterClientMock, clusterClientInstanceMock } from './core_service.test.mocks'; +import { + MockLegacyScopedClusterClient, + MockElasticsearchClient, + legacyClusterClientInstanceMock, +} from './core_service.test.mocks'; import Boom from 'boom'; import { Request } from 'hapi'; import { errors as esErrors } from 'elasticsearch'; import { LegacyElasticsearchErrorHelpers } from '../../elasticsearch/legacy'; +import { elasticsearchClientMock } from '../../elasticsearch/client/mocks'; +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import * as kbnTestServer from '../../../../test_utils/kbn_server'; +import { InternalElasticsearchServiceStart } from '../../elasticsearch'; interface User { id: string; @@ -44,6 +51,17 @@ const cookieOptions = { }; describe('http service', () => { + let esClient: ReturnType; + + beforeEach(async () => { + esClient = elasticsearchClientMock.createInternalClient(); + MockElasticsearchClient.mockImplementation(() => esClient); + }, 30000); + + afterEach(async () => { + MockElasticsearchClient.mockClear(); + }); + describe('auth', () => { let root: ReturnType; beforeEach(async () => { @@ -200,7 +218,7 @@ describe('http service', () => { }, 30000); afterEach(async () => { - clusterClientMock.mockClear(); + MockLegacyScopedClusterClient.mockClear(); await root.shutdown(); }); @@ -363,7 +381,7 @@ describe('http service', () => { }, 30000); afterEach(async () => { - clusterClientMock.mockClear(); + MockLegacyScopedClusterClient.mockClear(); await root.shutdown(); }); @@ -386,7 +404,7 @@ describe('http service', () => { await kbnTestServer.request.get(root, '/new-platform/').expect(200); // client contains authHeaders for BWC with legacy platform. - const [client] = clusterClientMock.mock.calls; + const [client] = MockLegacyScopedClusterClient.mock.calls; const [, , clientHeaders] = client; expect(clientHeaders).toEqual(authHeaders); }); @@ -410,7 +428,7 @@ describe('http service', () => { .set('Authorization', authorizationHeader) .expect(200); - const [client] = clusterClientMock.mock.calls; + const [client] = MockLegacyScopedClusterClient.mock.calls; const [, , clientHeaders] = client; expect(clientHeaders).toEqual({ authorization: authorizationHeader }); }); @@ -426,7 +444,7 @@ describe('http service', () => { }) ); - clusterClientInstanceMock.callAsCurrentUser.mockRejectedValue(authenticationError); + legacyClusterClientInstanceMock.callAsCurrentUser.mockRejectedValue(authenticationError); const router = createRouter('/new-platform'); router.get({ path: '/', validate: false }, async (context, req, res) => { @@ -441,4 +459,91 @@ describe('http service', () => { expect(response.header['www-authenticate']).toEqual('authenticate header'); }); }); + + describe('elasticsearch client', () => { + let root: ReturnType; + + beforeEach(async () => { + root = kbnTestServer.createRoot({ plugins: { initialize: false } }); + }, 30000); + + afterEach(async () => { + MockElasticsearchClient.mockClear(); + await root.shutdown(); + }); + + it('forwards unauthorized errors from elasticsearch', async () => { + const { http } = await root.setup(); + const { createRouter } = http; + // eslint-disable-next-line prefer-const + let elasticsearch: InternalElasticsearchServiceStart; + + esClient.ping.mockImplementation(() => + elasticsearchClientMock.createClientError( + new ResponseError({ + statusCode: 401, + body: { + error: { + type: 'Unauthorized', + }, + }, + warnings: [], + headers: { + 'WWW-Authenticate': 'content', + }, + meta: {} as any, + }) + ) + ); + + const router = createRouter('/new-platform'); + router.get({ path: '/', validate: false }, async (context, req, res) => { + await elasticsearch.client.asScoped(req).asInternalUser.ping(); + return res.ok(); + }); + + const coreStart = await root.start(); + elasticsearch = coreStart.elasticsearch; + + const { header } = await kbnTestServer.request.get(root, '/new-platform/').expect(401); + + expect(header['www-authenticate']).toEqual('content'); + }); + + it('uses a default value for `www-authenticate` header when ES 401 does not specify it', async () => { + const { http } = await root.setup(); + const { createRouter } = http; + // eslint-disable-next-line prefer-const + let elasticsearch: InternalElasticsearchServiceStart; + + esClient.ping.mockImplementation(() => + elasticsearchClientMock.createClientError( + new ResponseError({ + statusCode: 401, + body: { + error: { + type: 'Unauthorized', + }, + }, + warnings: [], + headers: {}, + meta: {} as any, + }) + ) + ); + + const router = createRouter('/new-platform'); + router.get({ path: '/', validate: false }, async (context, req, res) => { + await elasticsearch.client.asScoped(req).asInternalUser.ping(); + return res.ok(); + }); + + const coreStart = await root.start(); + elasticsearch = coreStart.elasticsearch; + + const { header } = await kbnTestServer.request.get(root, '/new-platform/').expect(401); + + expect(header['www-authenticate']).toEqual('Basic realm="Authorization Required"'); + }); + }); }); diff --git a/src/core/server/http/router/router.ts b/src/core/server/http/router/router.ts index 35eec746163ce..cc5279a396163 100644 --- a/src/core/server/http/router/router.ts +++ b/src/core/server/http/router/router.ts @@ -23,8 +23,17 @@ import Boom from 'boom'; import { isConfigSchema } from '@kbn/config-schema'; import { Logger } from '../../logging'; import { LegacyElasticsearchErrorHelpers } from '../../elasticsearch/legacy/errors'; +import { + isUnauthorizedError as isElasticsearchUnauthorizedError, + UnauthorizedError as EsNotAuthorizedError, +} from '../../elasticsearch/client/errors'; import { KibanaRequest } from './request'; -import { KibanaResponseFactory, kibanaResponseFactory, IKibanaResponse } from './response'; +import { + KibanaResponseFactory, + kibanaResponseFactory, + IKibanaResponse, + ErrorHttpResponseOptions, +} from './response'; import { RouteConfig, RouteConfigOptions, RouteMethod, validBodyOutput } from './route'; import { HapiResponseAdapter } from './response_adapter'; import { RequestHandlerContext } from '../../../server'; @@ -264,7 +273,13 @@ export class Router implements IRouter { return hapiResponseAdapter.handle(kibanaResponse); } catch (e) { this.log.error(e); - // forward 401 (boom) error from ES + // forward 401 errors from ES client + if (isElasticsearchUnauthorizedError(e)) { + return hapiResponseAdapter.handle( + kibanaResponseFactory.unauthorized(convertEsUnauthorized(e)) + ); + } + // forward 401 (boom) errors from legacy ES client if (LegacyElasticsearchErrorHelpers.isNotAuthorizedError(e)) { return e; } @@ -273,6 +288,21 @@ export class Router implements IRouter { } } +const convertEsUnauthorized = (e: EsNotAuthorizedError): ErrorHttpResponseOptions => { + const getAuthenticateHeaderValue = () => { + const header = Object.entries(e.headers).find( + ([key]) => key.toLowerCase() === 'www-authenticate' + ); + return header ? header[1] : 'Basic realm="Authorization Required"'; + }; + return { + body: e.message, + headers: { + 'www-authenticate': getAuthenticateHeaderValue(), + }, + }; +}; + type WithoutHeadArgument = T extends (first: any, ...rest: infer Params) => infer Return ? (...rest: Params) => Return : never; From c5073f4849409437bf73b8ec87616c8c98e2b59e Mon Sep 17 00:00:00 2001 From: cachedout Date: Tue, 21 Jul 2020 12:08:54 +0200 Subject: [PATCH 002/104] Archive e2e test results in ES (#72575) * Archive e2e test results in ES * Disable flaky comment feature and PR notifications * Update .ci/end2end.groovy Co-authored-by: Victor Martinez Co-authored-by: Victor Martinez --- .ci/end2end.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/end2end.groovy b/.ci/end2end.groovy index 97099c6f87448..ee117d362d59b 100644 --- a/.ci/end2end.groovy +++ b/.ci/end2end.groovy @@ -110,6 +110,9 @@ pipeline { archiveArtifacts(allowEmptyArchive: true, artifacts: "${E2E_DIR}/kibana.log") } } + cleanup { + notifyBuildResult(notifyPRComment: false, analyzeFlakey: false, shouldNotify: false) + } } } From 81cbd13db49888d72ba75723d4c41947a0733e3e Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Tue, 21 Jul 2020 12:17:01 +0100 Subject: [PATCH 003/104] chore(NA): fix grunt task for test:coverage (#72539) --- tasks/test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/test.js b/tasks/test.js index 96ec4d91db325..09821b97fe2e8 100644 --- a/tasks/test.js +++ b/tasks/test.js @@ -48,7 +48,7 @@ module.exports = function (grunt) { grunt.task.run(['run:karmaTestServer', ...ciShardTasks]); }); - grunt.registerTask('test:coverage', ['run:testCoverageServer', 'karma:coverage']); + grunt.registerTask('test:coverage', ['run:karmaTestCoverageServer', 'karma:coverage']); grunt.registerTask('test:quick', [ 'checkPlugins', From e1ffcccb9681e24c92d2048b90b3d38da9e3980e Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Tue, 21 Jul 2020 14:45:51 +0300 Subject: [PATCH 004/104] Add inspector for VEGA (#70941) * [WIP] Add inspector for VEGA Closes: #31189 * view -> dataset * cleanup * add spec viewer * cleanup code * use rx to retrieve data from adapters * Make custom inspector adapters registerable from the visType * fix flex-box size * cleanup * remove visTypesWithoutInspector from visualize_embeddable * fix PR comments * add vega folder to sass-lint * fix jest * Update src/plugins/vis_type_vega/public/vega_inspector/components/data_viewer.tsx Co-authored-by: Wylie Conlon * use addSignalListener * cleanup * add onColumnResize handler * EuiCodeEditor -> CodeEditor * fix type_check * fix issue with pagination * fix extra vertical scroll * add area-label for EuiButtonIcon * add area-label for EuiComboBox * Design Commit - Fixing up layout trying to remove any `.eui` classes and uses flex instead of percentage - Fixing text to use `Sentence case` not `Title Case` * Wrapper around signal viewer table * fix Jest snapshot Co-authored-by: Elastic Machine Co-authored-by: Wylie Conlon Co-authored-by: cchaos --- .sass-lint.yml | 1 + src/plugins/data/public/public.api.md | 3 +- .../data/public/search/aggs/buckets/terms.ts | 2 +- .../data/public/search/expressions/esaggs.ts | 2 +- .../utils/courier_inspector_stats.ts | 7 +- src/plugins/data/server/server.api.md | 1 - .../public/application/angular/discover.js | 2 +- .../embeddable/search_embeddable.ts | 2 +- .../expressions/common/execution/execution.ts | 4 +- .../expressions/common/execution/types.ts | 6 +- src/plugins/expressions/public/loader.ts | 5 +- .../public/input_control_vis_type.ts | 1 + .../inspector/common/adapters/index.ts | 9 +- .../common/adapters/request/index.ts | 1 + .../inspector/common/adapters/types.ts | 25 +++ src/plugins/inspector/public/index.scss | 2 +- src/plugins/inspector/public/plugin.tsx | 3 +- src/plugins/inspector/public/types.ts | 12 +- .../inspector_panel.test.tsx.snap | 6 +- .../inspector/public/ui/inspector_panel.scss | 12 ++ .../public/ui/inspector_panel.test.tsx | 3 +- .../inspector/public/ui/inspector_panel.tsx | 8 +- .../inspector/public/view_registry.test.ts | 2 +- src/plugins/inspector/public/view_registry.ts | 3 +- .../views/data/components/data_view.tsx | 3 +- .../inspector/public/views/data/index.tsx | 3 +- .../inspector/public/views/requests/index.ts | 3 +- .../vis_type_markdown/public/markdown_vis.ts | 1 + .../public/timelion_vis_type.tsx | 1 + .../public/metrics_type.ts | 1 + src/plugins/vis_type_vega/kibana.json | 2 +- .../vis_type_vega/public/_vega_vis.scss | 42 ++--- .../public/data_model/search_api.ts | 36 ++++- .../public/data_model/vega_parser.test.js | 1 + .../public/data_model/vega_parser.ts | 6 +- src/plugins/vis_type_vega/public/plugin.ts | 17 +- src/plugins/vis_type_vega/public/vega_fn.ts | 18 ++- .../vega_inspector/components/data_viewer.tsx | 114 ++++++++++++++ .../public/vega_inspector/components/index.ts | 22 +++ .../components/inspector_data_grid.tsx | 144 +++++++++++++++++ .../components/signal_viewer.tsx | 72 +++++++++ .../vega_inspector/components/spec_viewer.tsx | 97 ++++++++++++ .../public/vega_inspector/index.ts | 24 +++ .../public/vega_inspector/vega_adapter.ts | 148 ++++++++++++++++++ .../vega_inspector/vega_data_inspector.scss | 18 +++ .../vega_inspector/vega_data_inspector.tsx | 74 +++++++++ .../public/vega_inspector/vega_inspector.tsx | 57 +++++++ .../public/vega_request_handler.ts | 11 +- src/plugins/vis_type_vega/public/vega_type.ts | 4 +- .../public/vega_view/vega_base_view.js | 5 + .../public/vega_view/vega_map_view.js | 2 +- .../public/vega_view/vega_view.js | 2 +- .../public/embeddable/visualize_embeddable.ts | 14 +- .../public/vis_types/base_vis_type.ts | 4 + test/functional/apps/visualize/_vega_chart.js | 5 - x-pack/plugins/maps/public/kibana_services.js | 2 +- 56 files changed, 992 insertions(+), 83 deletions(-) create mode 100644 src/plugins/inspector/common/adapters/types.ts create mode 100644 src/plugins/inspector/public/ui/inspector_panel.scss create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/components/data_viewer.tsx create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/components/index.ts create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/components/inspector_data_grid.tsx create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/components/signal_viewer.tsx create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/components/spec_viewer.tsx create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/index.ts create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/vega_adapter.ts create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/vega_data_inspector.scss create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/vega_data_inspector.tsx create mode 100644 src/plugins/vis_type_vega/public/vega_inspector/vega_inspector.tsx diff --git a/.sass-lint.yml b/.sass-lint.yml index 50cbe81cc7da2..d6eaaf391de1a 100644 --- a/.sass-lint.yml +++ b/.sass-lint.yml @@ -3,6 +3,7 @@ files: - 'src/legacy/core_plugins/metrics/**/*.s+(a|c)ss' - 'src/plugins/timelion/**/*.s+(a|c)ss' - 'src/plugins/vis_type_vislib/**/*.s+(a|c)ss' + - 'src/plugins/vis_type_vega/**/*.s+(a|c)ss' - 'src/plugins/vis_type_xy/**/*.s+(a|c)ss' - 'x-pack/plugins/canvas/**/*.s+(a|c)ss' - 'x-pack/plugins/triggers_actions_ui/**/*.s+(a|c)ss' diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 38e0416233e25..a8868c07061c3 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -52,7 +52,6 @@ import { EuiButtonEmptyProps } from '@elastic/eui'; import { EuiComboBoxProps } from '@elastic/eui'; import { EuiConfirmModalProps } from '@elastic/eui'; import { EuiGlobalToastListToast } from '@elastic/eui'; -import { EventEmitter } from 'events'; import { ExclusiveUnion } from '@elastic/eui'; import { ExistsParams } from 'elasticsearch'; import { ExplainParams } from 'elasticsearch'; @@ -148,7 +147,7 @@ import { ReindexRethrottleParams } from 'elasticsearch'; import { RenderSearchTemplateParams } from 'elasticsearch'; import { Reporter } from '@kbn/analytics'; import { RequestAdapter } from 'src/plugins/inspector/common'; -import { RequestStatistics as RequestStatistics_2 } from 'src/plugins/inspector/common'; +import { RequestStatistics } from 'src/plugins/inspector/common'; import { Required } from '@kbn/utility-types'; import * as Rx from 'rxjs'; import { SavedObject } from 'src/core/server'; diff --git a/src/plugins/data/public/search/aggs/buckets/terms.ts b/src/plugins/data/public/search/aggs/buckets/terms.ts index d3acd33d73d01..5c8483cf21369 100644 --- a/src/plugins/data/public/search/aggs/buckets/terms.ts +++ b/src/plugins/data/public/search/aggs/buckets/terms.ts @@ -129,7 +129,7 @@ export const getTermsBucketAgg = () => const response = await nestedSearchSource.fetch({ abortSignal }); request - .stats(getResponseInspectorStats(nestedSearchSource, response)) + .stats(getResponseInspectorStats(response, nestedSearchSource)) .ok({ json: response }); resp = mergeOtherBucketAggResponse(aggConfigs, resp, response, aggConfig, filterAgg()); } diff --git a/src/plugins/data/public/search/expressions/esaggs.ts b/src/plugins/data/public/search/expressions/esaggs.ts index b01f17762b2be..690f6b1df11c3 100644 --- a/src/plugins/data/public/search/expressions/esaggs.ts +++ b/src/plugins/data/public/search/expressions/esaggs.ts @@ -160,7 +160,7 @@ const handleCourierRequest = async ({ (searchSource as any).lastQuery = queryHash; - request.stats(getResponseInspectorStats(searchSource, response)).ok({ json: response }); + request.stats(getResponseInspectorStats(response, searchSource)).ok({ json: response }); (searchSource as any).rawResponse = response; } catch (e) { diff --git a/src/plugins/data/public/search/expressions/utils/courier_inspector_stats.ts b/src/plugins/data/public/search/expressions/utils/courier_inspector_stats.ts index 96d0aaa16f6ba..c933e8cd3e961 100644 --- a/src/plugins/data/public/search/expressions/utils/courier_inspector_stats.ts +++ b/src/plugins/data/public/search/expressions/utils/courier_inspector_stats.ts @@ -61,10 +61,11 @@ export function getRequestInspectorStats(searchSource: ISearchSource) { /** @public */ export function getResponseInspectorStats( - searchSource: ISearchSource, - resp: SearchResponse + resp: SearchResponse, + searchSource?: ISearchSource ) { - const lastRequest = searchSource.history && searchSource.history[searchSource.history.length - 1]; + const lastRequest = + searchSource?.history && searchSource.history[searchSource.history.length - 1]; const stats: RequestStatistics = {}; if (resp && resp.took) { diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index c5d19fef9531e..99a77ff9aeb10 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -39,7 +39,6 @@ import { DeleteTemplateParams } from 'elasticsearch'; import { DetailedPeerCertificate } from 'tls'; import { Duration } from 'moment'; import { ErrorToastOptions } from 'src/core/public/notifications'; -import { EventEmitter } from 'events'; import { ExistsParams } from 'elasticsearch'; import { ExplainParams } from 'elasticsearch'; import { FieldStatsParams } from 'elasticsearch'; diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 9b8b32b51cfd8..c791bdd850151 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -874,7 +874,7 @@ function discoverController( } function onResults(resp) { - inspectorRequest.stats(getResponseInspectorStats($scope.searchSource, resp)).ok({ json: resp }); + inspectorRequest.stats(getResponseInspectorStats(resp, $scope.searchSource)).ok({ json: resp }); if (getTimeField()) { const tabifiedData = tabifyAggResponse($scope.vis.data.aggs, resp); diff --git a/src/plugins/discover/public/application/embeddable/search_embeddable.ts b/src/plugins/discover/public/application/embeddable/search_embeddable.ts index 9a3dd0d310ff7..b621017677c58 100644 --- a/src/plugins/discover/public/application/embeddable/search_embeddable.ts +++ b/src/plugins/discover/public/application/embeddable/search_embeddable.ts @@ -307,7 +307,7 @@ export class SearchEmbeddable extends Embeddable this.updateOutput({ loading: false, error: undefined }); // Log response to inspector - inspectorRequest.stats(getResponseInspectorStats(searchSource, resp)).ok({ json: resp }); + inspectorRequest.stats(getResponseInspectorStats(resp, searchSource)).ok({ json: resp }); // Apply the changes to the angular scope this.searchScope.$apply(() => { diff --git a/src/plugins/expressions/common/execution/execution.ts b/src/plugins/expressions/common/execution/execution.ts index f42ee18965309..3533500a2fbc5 100644 --- a/src/plugins/expressions/common/execution/execution.ts +++ b/src/plugins/expressions/common/execution/execution.ts @@ -23,7 +23,7 @@ import { createExecutionContainer, ExecutionContainer } from './container'; import { createError } from '../util'; import { Defer, now } from '../../../kibana_utils/common'; import { toPromise } from '../../../data/common/utils/abort_utils'; -import { RequestAdapter, DataAdapter } from '../../../inspector/common'; +import { RequestAdapter, DataAdapter, Adapters } from '../../../inspector/common'; import { isExpressionValueError, ExpressionValueError } from '../expression_types/specs/error'; import { ExpressionAstExpression, @@ -70,7 +70,7 @@ export class Execution< ExtraContext extends Record = Record, Input = unknown, Output = unknown, - InspectorAdapters = ExtraContext['inspectorAdapters'] extends object + InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object ? ExtraContext['inspectorAdapters'] : DefaultInspectorAdapters > { diff --git a/src/plugins/expressions/common/execution/types.ts b/src/plugins/expressions/common/execution/types.ts index 51538394cd125..7c26e586fb790 100644 --- a/src/plugins/expressions/common/execution/types.ts +++ b/src/plugins/expressions/common/execution/types.ts @@ -18,7 +18,7 @@ */ import { ExpressionType } from '../expression_types'; -import { DataAdapter, RequestAdapter } from '../../../inspector/common'; +import { Adapters, DataAdapter, RequestAdapter } from '../../../inspector/common'; import { TimeRange, Query, Filter } from '../../../data/common'; import { SavedObject, SavedObjectAttributes } from '../../../../core/public'; @@ -26,7 +26,7 @@ import { SavedObject, SavedObjectAttributes } from '../../../../core/public'; * `ExecutionContext` is an object available to all functions during a single execution; * it provides various methods to perform side-effects. */ -export interface ExecutionContext { +export interface ExecutionContext { /** * Get initial input with which execution started. */ @@ -75,7 +75,7 @@ export interface ExecutionContext { /** * Adapters used to open the inspector. */ - adapters: Adapters; + adapters: TAdapters; /** * The title that the inspector is currently using e.g. a visualization name. */ diff --git a/src/plugins/inspector/public/ui/__snapshots__/inspector_panel.test.tsx.snap b/src/plugins/inspector/public/ui/__snapshots__/inspector_panel.test.tsx.snap index 0e9560cbd7962..9ed4e60cac519 100644 --- a/src/plugins/inspector/public/ui/__snapshots__/inspector_panel.test.tsx.snap +++ b/src/plugins/inspector/public/ui/__snapshots__/inspector_panel.test.tsx.snap @@ -306,9 +306,11 @@ exports[`InspectorPanel should render as expected 1`] = ` - +
div { + display: flex; + flex-direction: column; + + > div { + flex-grow: 1; + } + } +} diff --git a/src/plugins/inspector/public/ui/inspector_panel.test.tsx b/src/plugins/inspector/public/ui/inspector_panel.test.tsx index c482b6fa8033b..23f698c23793b 100644 --- a/src/plugins/inspector/public/ui/inspector_panel.test.tsx +++ b/src/plugins/inspector/public/ui/inspector_panel.test.tsx @@ -20,7 +20,8 @@ import React from 'react'; import { mountWithIntl } from 'test_utils/enzyme_helpers'; import { InspectorPanel } from './inspector_panel'; -import { Adapters, InspectorViewDescription } from '../types'; +import { InspectorViewDescription } from '../types'; +import { Adapters } from '../../common'; describe('InspectorPanel', () => { let adapters: Adapters; diff --git a/src/plugins/inspector/public/ui/inspector_panel.tsx b/src/plugins/inspector/public/ui/inspector_panel.tsx index 85705b6b74f55..37a51257112d6 100644 --- a/src/plugins/inspector/public/ui/inspector_panel.tsx +++ b/src/plugins/inspector/public/ui/inspector_panel.tsx @@ -17,11 +17,13 @@ * under the License. */ +import './inspector_panel.scss'; import { i18n } from '@kbn/i18n'; import React, { Component } from 'react'; import PropTypes from 'prop-types'; import { EuiFlexGroup, EuiFlexItem, EuiFlyoutHeader, EuiTitle, EuiFlyoutBody } from '@elastic/eui'; -import { Adapters, InspectorViewDescription } from '../types'; +import { InspectorViewDescription } from '../types'; +import { Adapters } from '../../common'; import { InspectorViewChooser } from './inspector_view_chooser'; function hasAdaptersChanged(oldAdapters: Adapters, newAdapters: Adapters) { @@ -122,7 +124,9 @@ export class InspectorPanel extends Component - {this.renderSelectedPanel()} + + {this.renderSelectedPanel()} + ); } diff --git a/src/plugins/inspector/public/view_registry.test.ts b/src/plugins/inspector/public/view_registry.test.ts index 542328d4f48da..13e109f50243c 100644 --- a/src/plugins/inspector/public/view_registry.test.ts +++ b/src/plugins/inspector/public/view_registry.test.ts @@ -20,7 +20,7 @@ import { InspectorViewRegistry } from './view_registry'; import { InspectorViewDescription } from './types'; -import { Adapters } from './types'; +import { Adapters } from '../common'; function createMockView( params: { diff --git a/src/plugins/inspector/public/view_registry.ts b/src/plugins/inspector/public/view_registry.ts index 800d917af28ca..be84a62a11712 100644 --- a/src/plugins/inspector/public/view_registry.ts +++ b/src/plugins/inspector/public/view_registry.ts @@ -18,7 +18,8 @@ */ import { EventEmitter } from 'events'; -import { Adapters, InspectorViewDescription } from './types'; +import { InspectorViewDescription } from './types'; +import { Adapters } from '../common'; /** * @callback viewShouldShowFunc diff --git a/src/plugins/inspector/public/views/data/components/data_view.tsx b/src/plugins/inspector/public/views/data/components/data_view.tsx index e03c165d96a27..1a2b6f9922d2d 100644 --- a/src/plugins/inspector/public/views/data/components/data_view.tsx +++ b/src/plugins/inspector/public/views/data/components/data_view.tsx @@ -30,7 +30,8 @@ import { } from '@elastic/eui'; import { DataTableFormat } from './data_table'; -import { InspectorViewProps, Adapters } from '../../../types'; +import { InspectorViewProps } from '../../../types'; +import { Adapters } from '../../../../common'; import { TabularLoaderOptions, TabularData, diff --git a/src/plugins/inspector/public/views/data/index.tsx b/src/plugins/inspector/public/views/data/index.tsx index 0cd88442bf8f8..b02e02bbe6b6b 100644 --- a/src/plugins/inspector/public/views/data/index.tsx +++ b/src/plugins/inspector/public/views/data/index.tsx @@ -20,7 +20,8 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; import { DataViewComponent } from './components/data_view'; -import { Adapters, InspectorViewDescription, InspectorViewProps } from '../../types'; +import { InspectorViewDescription, InspectorViewProps } from '../../types'; +import { Adapters } from '../../../common'; import { IUiSettingsClient } from '../../../../../core/public'; export const getDataViewDescription = ( diff --git a/src/plugins/inspector/public/views/requests/index.ts b/src/plugins/inspector/public/views/requests/index.ts index 741da76872710..00a223e1e30fa 100644 --- a/src/plugins/inspector/public/views/requests/index.ts +++ b/src/plugins/inspector/public/views/requests/index.ts @@ -19,7 +19,8 @@ import { i18n } from '@kbn/i18n'; import { RequestsViewComponent } from './components/requests_view'; -import { Adapters, InspectorViewDescription } from '../../types'; +import { InspectorViewDescription } from '../../types'; +import { Adapters } from '../../../common'; export const getRequestsViewDescription = (): InspectorViewDescription => ({ title: i18n.translate('inspector.requests.requestsTitle', { diff --git a/src/plugins/vis_type_markdown/public/markdown_vis.ts b/src/plugins/vis_type_markdown/public/markdown_vis.ts index 3309330d7527c..089e00bb44937 100644 --- a/src/plugins/vis_type_markdown/public/markdown_vis.ts +++ b/src/plugins/vis_type_markdown/public/markdown_vis.ts @@ -66,4 +66,5 @@ export const markdownVisDefinition = { }, requestHandler: 'none', responseHandler: 'none', + inspectorAdapters: {}, }; diff --git a/src/plugins/vis_type_timelion/public/timelion_vis_type.tsx b/src/plugins/vis_type_timelion/public/timelion_vis_type.tsx index 52addb3c2d9d2..b4c90700b160f 100644 --- a/src/plugins/vis_type_timelion/public/timelion_vis_type.tsx +++ b/src/plugins/vis_type_timelion/public/timelion_vis_type.tsx @@ -62,6 +62,7 @@ export function getTimelionVisDefinition(dependencies: TimelionVisDependencies) }, requestHandler: timelionRequestHandler, responseHandler: 'none', + inspectorAdapters: {}, options: { showIndexSelection: false, showQueryBar: false, diff --git a/src/plugins/vis_type_timeseries/public/metrics_type.ts b/src/plugins/vis_type_timeseries/public/metrics_type.ts index 649ee765cc642..44b0334a37871 100644 --- a/src/plugins/vis_type_timeseries/public/metrics_type.ts +++ b/src/plugins/vis_type_timeseries/public/metrics_type.ts @@ -78,5 +78,6 @@ export const metricsVisDefinition = { showIndexSelection: false, }, requestHandler: metricsRequestHandler, + inspectorAdapters: {}, responseHandler: 'none', }; diff --git a/src/plugins/vis_type_vega/kibana.json b/src/plugins/vis_type_vega/kibana.json index d7a92de627a99..7ba5f23f10564 100644 --- a/src/plugins/vis_type_vega/kibana.json +++ b/src/plugins/vis_type_vega/kibana.json @@ -3,6 +3,6 @@ "version": "kibana", "server": true, "ui": true, - "requiredPlugins": ["data", "visualizations", "mapsLegacy", "expressions"], + "requiredPlugins": ["data", "visualizations", "mapsLegacy", "expressions", "inspector"], "requiredBundles": ["kibanaUtils", "kibanaReact"] } diff --git a/src/plugins/vis_type_vega/public/_vega_vis.scss b/src/plugins/vis_type_vega/public/_vega_vis.scss index 4fc6fbc326ec1..f9468d677eeed 100644 --- a/src/plugins/vis_type_vega/public/_vega_vis.scss +++ b/src/plugins/vis_type_vega/public/_vega_vis.scss @@ -17,6 +17,7 @@ // BUG #23514: Make sure Vega doesn't display the controls in two places .vega-bindings { + // sass-lint:disable no-important display: none !important; } } @@ -47,7 +48,7 @@ width: $euiSizeM * 10 - $euiSize; } - input[type="range"] { + input[type='range'] { width: $euiSizeM * 10; display: inline-block; vertical-align: middle; @@ -74,7 +75,7 @@ top: 0; width: 100%; margin: auto; - opacity: 0.8; + opacity: .8; z-index: 1; list-style: none; } @@ -115,25 +116,30 @@ @include euiTextTruncate; padding-top: $euiSizeXS; padding-bottom: $euiSizeXS; - } - td.key { - color: $euiColorMediumShade; - max-width: $euiSize * 10; - text-align: right; - padding-right: $euiSizeXS; - } - td.value { - max-width: $euiSizeL * 10; - text-align: left; - } + &.key { + color: $euiColorMediumShade; + max-width: $euiSize * 10; + text-align: right; + padding-right: $euiSizeXS; + } - @media only screen and (max-width: map-get($euiBreakpoints, 'm')){ - td.key { - max-width: $euiSize * 6; + &.value { + max-width: $euiSizeL * 10; + text-align: left; } - td.value { - max-width: $euiSize * 10; + } + + + @media only screen and (max-width: map-get($euiBreakpoints, 'm')) { + td { + &.key { + max-width: $euiSize * 6; + } + + &.value { + max-width: $euiSize * 10; + } } } } diff --git a/src/plugins/vis_type_vega/public/data_model/search_api.ts b/src/plugins/vis_type_vega/public/data_model/search_api.ts index c2eecf13c2d51..18387a6ab0876 100644 --- a/src/plugins/vis_type_vega/public/data_model/search_api.ts +++ b/src/plugins/vis_type_vega/public/data_model/search_api.ts @@ -18,13 +18,17 @@ */ import { combineLatest } from 'rxjs'; -import { map } from 'rxjs/operators'; +import { map, tap } from 'rxjs/operators'; import { CoreStart, IUiSettingsClient } from 'kibana/public'; import { getSearchParamsFromRequest, SearchRequest, DataPublicPluginStart, + IEsSearchResponse, } from '../../../data/public'; +import { search as dataPluginSearch } from '../../../data/public'; +import { VegaInspectorAdapters } from '../vega_inspector'; +import { RequestResponder } from '../../../inspector/public'; export interface SearchAPIDependencies { uiSettings: IUiSettingsClient; @@ -35,26 +39,52 @@ export interface SearchAPIDependencies { export class SearchAPI { constructor( private readonly dependencies: SearchAPIDependencies, - private readonly abortSignal?: AbortSignal + private readonly abortSignal?: AbortSignal, + public readonly inspectorAdapters?: VegaInspectorAdapters ) {} search(searchRequests: SearchRequest[]) { const { search } = this.dependencies.search; + const requestResponders: any = {}; return combineLatest( searchRequests.map((request, index) => { + const requestId: number = index; const params = getSearchParamsFromRequest(request, { uiSettings: this.dependencies.uiSettings, injectedMetadata: this.dependencies.injectedMetadata, }); + if (this.inspectorAdapters) { + requestResponders[requestId] = this.inspectorAdapters.requests.start( + `#${requestId}`, + request + ); + requestResponders[requestId].json(params.body); + } + return search({ params }, { signal: this.abortSignal }).pipe( + tap((data) => this.inspectSearchResult(data, requestResponders[requestId])), map((data) => ({ - id: index, + id: requestId, rawResponse: data.rawResponse, })) ); }) ); } + + public resetSearchStats() { + if (this.inspectorAdapters) { + this.inspectorAdapters.requests.reset(); + } + } + + private inspectSearchResult(response: IEsSearchResponse, requestResponder: RequestResponder) { + if (requestResponder) { + requestResponder + .stats(dataPluginSearch.getResponseInspectorStats(response.rawResponse)) + .ok({ json: response.rawResponse }); + } + } } diff --git a/src/plugins/vis_type_vega/public/data_model/vega_parser.test.js b/src/plugins/vis_type_vega/public/data_model/vega_parser.test.js index 51aa4313a97b5..e29e16e3212f4 100644 --- a/src/plugins/vis_type_vega/public/data_model/vega_parser.test.js +++ b/src/plugins/vis_type_vega/public/data_model/vega_parser.test.js @@ -97,6 +97,7 @@ describe('VegaParser._resolveEsQueries', () => { search: jest.fn(() => ({ toPromise: jest.fn(() => Promise.resolve(data)), })), + resetSearchStats: jest.fn(), }; }); diff --git a/src/plugins/vis_type_vega/public/data_model/vega_parser.ts b/src/plugins/vis_type_vega/public/data_model/vega_parser.ts index 17166e1540755..c867523d2b3b3 100644 --- a/src/plugins/vis_type_vega/public/data_model/vega_parser.ts +++ b/src/plugins/vis_type_vega/public/data_model/vega_parser.ts @@ -79,6 +79,7 @@ export class VegaParser { paddingHeight?: number; containerDir?: ControlsLocation | ControlsDirection; controlsDir?: ControlsLocation; + searchAPI: SearchAPI; constructor( spec: VegaSpec | string, @@ -92,10 +93,11 @@ export class VegaParser { this.error = undefined; this.warnings = []; + this.searchAPI = searchAPI; const onWarn = this._onWarning.bind(this); this._urlParsers = { - elasticsearch: new EsQueryParser(timeCache, searchAPI, filters, onWarn), + elasticsearch: new EsQueryParser(timeCache, this.searchAPI, filters, onWarn), emsfile: new EmsFileParser(serviceSettings), url: new UrlParser(onWarn), }; @@ -541,6 +543,8 @@ export class VegaParser { async _resolveDataUrls() { const pending: PendingType = {}; + this.searchAPI.resetSearchStats(); + this._findObjectDataUrls(this.spec!, (obj: Data) => { const url = obj.url; delete obj.url; diff --git a/src/plugins/vis_type_vega/public/plugin.ts b/src/plugins/vis_type_vega/public/plugin.ts index c20a104736291..00c6b2e3c8d5b 100644 --- a/src/plugins/vis_type_vega/public/plugin.ts +++ b/src/plugins/vis_type_vega/public/plugin.ts @@ -18,8 +18,10 @@ */ import { PluginInitializerContext, CoreSetup, CoreStart, Plugin } from '../../../core/public'; import { Plugin as ExpressionsPublicPlugin } from '../../expressions/public'; -import { Plugin as DataPublicPlugin } from '../../data/public'; +import { DataPublicPluginSetup, DataPublicPluginStart } from '../../data/public'; import { VisualizationsSetup } from '../../visualizations/public'; +import { Setup as InspectorSetup } from '../../inspector/public'; + import { setNotifications, setData, @@ -37,11 +39,13 @@ import { IServiceSettings } from '../../maps_legacy/public'; import './index.scss'; import { ConfigSchema } from '../config'; +import { getVegaInspectorView } from './vega_inspector'; + /** @internal */ export interface VegaVisualizationDependencies { core: CoreSetup; plugins: { - data: ReturnType; + data: DataPublicPluginSetup; }; serviceSettings: IServiceSettings; } @@ -50,13 +54,14 @@ export interface VegaVisualizationDependencies { export interface VegaPluginSetupDependencies { expressions: ReturnType; visualizations: VisualizationsSetup; - data: ReturnType; + inspector: InspectorSetup; + data: DataPublicPluginSetup; mapsLegacy: any; } /** @internal */ export interface VegaPluginStartDependencies { - data: ReturnType; + data: DataPublicPluginStart; } /** @internal */ @@ -69,7 +74,7 @@ export class VegaPlugin implements Plugin, void> { public async setup( core: CoreSetup, - { data, expressions, visualizations, mapsLegacy }: VegaPluginSetupDependencies + { inspector, data, expressions, visualizations, mapsLegacy }: VegaPluginSetupDependencies ) { setInjectedVars({ enableExternalUrls: this.initializerContext.config.get().enableExternalUrls, @@ -88,6 +93,8 @@ export class VegaPlugin implements Plugin, void> { serviceSettings: mapsLegacy.serviceSettings, }; + inspector.registerView(getVegaInspectorView({ uiSettings: core.uiSettings })); + expressions.registerFunction(() => createVegaFn(visualizationDependencies)); visualizations.createBaseVisualization(createVegaTypeDefinition(visualizationDependencies)); diff --git a/src/plugins/vis_type_vega/public/vega_fn.ts b/src/plugins/vis_type_vega/public/vega_fn.ts index d077aa7aee004..c109bb3c6e90c 100644 --- a/src/plugins/vis_type_vega/public/vega_fn.ts +++ b/src/plugins/vis_type_vega/public/vega_fn.ts @@ -19,9 +19,15 @@ import { get } from 'lodash'; import { i18n } from '@kbn/i18n'; -import { ExpressionFunctionDefinition, KibanaContext, Render } from '../../expressions/public'; +import { + ExecutionContext, + ExpressionFunctionDefinition, + KibanaContext, + Render, +} from '../../expressions/public'; import { VegaVisualizationDependencies } from './plugin'; import { createVegaRequestHandler } from './vega_request_handler'; +import { VegaInspectorAdapters } from './vega_inspector/index'; import { TimeRange, Query } from '../../data/public'; import { VegaParser } from './data_model/vega_parser'; @@ -42,7 +48,13 @@ interface RenderValue { export const createVegaFn = ( dependencies: VegaVisualizationDependencies -): ExpressionFunctionDefinition<'vega', Input, Arguments, Output> => ({ +): ExpressionFunctionDefinition< + 'vega', + Input, + Arguments, + Output, + ExecutionContext +> => ({ name: 'vega', type: 'render', inputTypes: ['kibana_context', 'null'], @@ -57,7 +69,7 @@ export const createVegaFn = ( }, }, async fn(input, args, context) { - const vegaRequestHandler = createVegaRequestHandler(dependencies, context.abortSignal); + const vegaRequestHandler = createVegaRequestHandler(dependencies, context); const response = await vegaRequestHandler({ timeRange: get(input, 'timeRange') as TimeRange, diff --git a/src/plugins/vis_type_vega/public/vega_inspector/components/data_viewer.tsx b/src/plugins/vis_type_vega/public/vega_inspector/components/data_viewer.tsx new file mode 100644 index 0000000000000..9b09a09eb05e0 --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/components/data_viewer.tsx @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React, { useState, useCallback, useEffect } from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiComboBox, + EuiFlexGroup, + EuiComboBoxProps, + EuiFlexItem, + EuiSpacer, + CommonProps, +} from '@elastic/eui'; +import { VegaAdapter, InspectDataSets } from '../vega_adapter'; +import { InspectorDataGrid } from './inspector_data_grid'; + +interface DataViewerProps extends CommonProps { + vegaAdapter: VegaAdapter; +} + +const getDataGridArialabel = (view: InspectDataSets) => + i18n.translate('visTypeVega.inspector.dataViewer.gridAriaLabel', { + defaultMessage: '{name} data grid', + values: { + name: view.id, + }, + }); + +const dataSetAriaLabel = i18n.translate('visTypeVega.inspector.dataViewer.dataSetAriaLabel', { + defaultMessage: 'Data set', +}); + +export const DataViewer = ({ vegaAdapter, ...rest }: DataViewerProps) => { + const [inspectDataSets, setInspectDataSets] = useState([]); + const [selectedView, setSelectedView] = useState(); + const [dataGridAriaLabel, setDataGridAriaLabel] = useState(''); + + const onViewChange: EuiComboBoxProps['onChange'] = useCallback( + (selectedOptions) => { + const newView = inspectDataSets!.find((view) => view.id === selectedOptions[0].label); + + if (newView) { + setDataGridAriaLabel(getDataGridArialabel(newView)); + setSelectedView(newView); + } + }, + [inspectDataSets] + ); + + useEffect(() => { + const subscription = vegaAdapter.getDataSetsSubscription().subscribe((dataSets) => { + setInspectDataSets(dataSets); + }); + + return () => { + subscription.unsubscribe(); + }; + }, [vegaAdapter]); + + useEffect(() => { + if (inspectDataSets) { + if (!selectedView) { + setSelectedView(inspectDataSets[0]); + } else { + setDataGridAriaLabel(getDataGridArialabel(selectedView)); + } + } + }, [selectedView, inspectDataSets]); + + if (!selectedView) { + return null; + } + + return ( + + + + ({ + label: item.id, + }))} + aria-label={dataSetAriaLabel} + onChange={onViewChange} + isClearable={false} + singleSelection={{ asPlainText: true }} + selectedOptions={[{ label: selectedView.id }]} + /> + + + + + + ); +}; diff --git a/src/plugins/vis_type_vega/public/vega_inspector/components/index.ts b/src/plugins/vis_type_vega/public/vega_inspector/components/index.ts new file mode 100644 index 0000000000000..76e631f9ecd94 --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/components/index.ts @@ -0,0 +1,22 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { DataViewer } from './data_viewer'; +export { SignalViewer } from './signal_viewer'; +export { SpecViewer } from './spec_viewer'; diff --git a/src/plugins/vis_type_vega/public/vega_inspector/components/inspector_data_grid.tsx b/src/plugins/vis_type_vega/public/vega_inspector/components/inspector_data_grid.tsx new file mode 100644 index 0000000000000..00f24e03d8196 --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/components/inspector_data_grid.tsx @@ -0,0 +1,144 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React, { useState, useCallback, useMemo, useEffect } from 'react'; +import { EuiDataGrid, EuiDataGridSorting, EuiDataGridProps } from '@elastic/eui'; +import { VegaRuntimeData } from '../vega_adapter'; + +const DEFAULT_PAGE_SIZE = 15; + +interface InspectorDataGridProps extends VegaRuntimeData { + dataGridAriaLabel: string; +} + +export const InspectorDataGrid = ({ columns, data, dataGridAriaLabel }: InspectorDataGridProps) => { + const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: DEFAULT_PAGE_SIZE }); + const onChangeItemsPerPage = useCallback( + (pageSize) => setPagination((p) => ({ ...p, pageSize, pageIndex: 0 })), + [setPagination] + ); + + const onChangePage = useCallback((pageIndex) => setPagination((p) => ({ ...p, pageIndex })), [ + setPagination, + ]); + + // Column visibility + const [visibleColumns, setVisibleColumns] = useState([]); + + useEffect( + () => { + setPagination({ + ...pagination, + pageIndex: 0, + }); + setVisibleColumns(columns.map((column) => column.id)); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [dataGridAriaLabel] + ); + + // Sorting + const [sortingColumns, setSortingColumns] = useState([]); + + const onSort = useCallback( + (newSortingColumns: EuiDataGridSorting['columns']) => { + setSortingColumns(newSortingColumns); + }, + [setSortingColumns] + ); + + let gridData = useMemo(() => { + return [...data].sort((a, b) => { + for (let i = 0; i < sortingColumns.length; i++) { + const column = sortingColumns[i]; + const aValue = a[column.id]; + const bValue = b[column.id]; + + if (aValue < bValue) return column.direction === 'asc' ? -1 : 1; + if (aValue > bValue) return column.direction === 'asc' ? 1 : -1; + } + return 0; + }); + }, [data, sortingColumns]); + + const renderCellValue = useMemo(() => { + return (({ rowIndex, columnId }) => { + let adjustedRowIndex = rowIndex; + + // If we are doing the pagination (instead of leaving that to the grid) + // then the row index must be adjusted as `data` has already been pruned to the page size + adjustedRowIndex = rowIndex - pagination.pageIndex * pagination.pageSize; + + return gridData.hasOwnProperty(adjustedRowIndex) + ? gridData[adjustedRowIndex][columnId] || null + : null; + }) as EuiDataGridProps['renderCellValue']; + }, [gridData, pagination.pageIndex, pagination.pageSize]); + + // Pagination + gridData = useMemo(() => { + const rowStart = pagination.pageIndex * pagination.pageSize; + const rowEnd = Math.min(rowStart + pagination.pageSize, gridData.length); + return gridData.slice(rowStart, rowEnd); + }, [gridData, pagination]); + + // Resize + const [columnsWidth, setColumnsWidth] = useState>({}); + + const onColumnResize: EuiDataGridProps['onColumnResize'] = useCallback( + ({ columnId, width }) => { + setColumnsWidth({ + ...columnsWidth, + [columnId]: width, + }); + }, + [columnsWidth] + ); + + return ( + { + if (columnsWidth[column.id]) { + return { + ...column, + initialWidth: columnsWidth[column.id], + }; + } + return column; + })} + columnVisibility={{ + visibleColumns, + setVisibleColumns, + }} + rowCount={data.length} + renderCellValue={renderCellValue} + sorting={{ columns: sortingColumns, onSort }} + toolbarVisibility={{ + showFullScreenSelector: false, + }} + onColumnResize={onColumnResize} + pagination={{ + ...pagination, + pageSizeOptions: [DEFAULT_PAGE_SIZE, 25, 50], + onChangeItemsPerPage, + onChangePage, + }} + /> + ); +}; diff --git a/src/plugins/vis_type_vega/public/vega_inspector/components/signal_viewer.tsx b/src/plugins/vis_type_vega/public/vega_inspector/components/signal_viewer.tsx new file mode 100644 index 0000000000000..39df004f327a4 --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/components/signal_viewer.tsx @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React, { useEffect, useState } from 'react'; + +import { EuiSpacer } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { VegaAdapter, InspectSignalsSets } from '../vega_adapter'; +import { InspectorDataGrid } from './inspector_data_grid'; + +interface SignalViewerProps { + vegaAdapter: VegaAdapter; +} + +const initialSignalColumnWidth = 150; + +const signalDataGridAriaLabel = i18n.translate('visTypeVega.inspector.signalViewer.gridAriaLabel', { + defaultMessage: 'Signal values data grid', +}); + +export const SignalViewer = ({ vegaAdapter }: SignalViewerProps) => { + const [inspectSignalsSets, setInspectSignalsSets] = useState(); + + useEffect(() => { + const subscription = vegaAdapter.getSignalsSetsSubscription().subscribe((signalSets) => { + if (signalSets) { + setInspectSignalsSets(signalSets); + } + }); + return () => { + subscription.unsubscribe(); + }; + }, [vegaAdapter]); + + if (!inspectSignalsSets) { + return null; + } + + return ( +
+ + { + if (index === 0) { + return { + ...column, + initialWidth: initialSignalColumnWidth, + }; + } + return column; + })} + data={inspectSignalsSets.data} + dataGridAriaLabel={signalDataGridAriaLabel} + /> +
+ ); +}; diff --git a/src/plugins/vis_type_vega/public/vega_inspector/components/spec_viewer.tsx b/src/plugins/vis_type_vega/public/vega_inspector/components/spec_viewer.tsx new file mode 100644 index 0000000000000..54f7974960aa2 --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/components/spec_viewer.tsx @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React, { useEffect, useState } from 'react'; +import { i18n } from '@kbn/i18n'; + +import { + EuiFlexItem, + EuiFlexGroup, + EuiCopy, + EuiButtonEmpty, + EuiSpacer, + CommonProps, +} from '@elastic/eui'; +import { VegaAdapter } from '../vega_adapter'; +import { CodeEditor } from '../../../../kibana_react/public'; + +interface SpecViewerProps extends CommonProps { + vegaAdapter: VegaAdapter; +} + +const copyToClipboardLabel = i18n.translate( + 'visTypeVega.inspector.specViewer.copyToClipboardLabel', + { + defaultMessage: 'Copy to clipboard', + } +); + +export const SpecViewer = ({ vegaAdapter, ...rest }: SpecViewerProps) => { + const [spec, setSpec] = useState(); + + useEffect(() => { + const subscription = vegaAdapter.getSpecSubscription().subscribe((data) => { + if (data) { + setSpec(data); + } + }); + return () => { + subscription.unsubscribe(); + }; + }, [vegaAdapter]); + + if (!spec) { + return null; + } + + return ( + + + +
+ + {(copy) => ( + + {copyToClipboardLabel} + + )} + +
+ + + {}} + options={{ + readOnly: true, + lineNumbers: 'off', + fontSize: 12, + minimap: { + enabled: false, + }, + scrollBeyondLastLine: false, + wordWrap: 'on', + wrappingIndent: 'indent', + automaticLayout: true, + }} + /> + + + ); +}; diff --git a/src/plugins/vis_type_vega/public/vega_inspector/index.ts b/src/plugins/vis_type_vega/public/vega_inspector/index.ts new file mode 100644 index 0000000000000..24da27d2d742d --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/index.ts @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { + createInspectorAdapters, + getVegaInspectorView, + VegaInspectorAdapters, +} from './vega_inspector'; diff --git a/src/plugins/vis_type_vega/public/vega_inspector/vega_adapter.ts b/src/plugins/vis_type_vega/public/vega_inspector/vega_adapter.ts new file mode 100644 index 0000000000000..e4c536af40591 --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/vega_adapter.ts @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Observable, ReplaySubject, fromEventPattern, merge, timer } from 'rxjs'; +import { map, switchMap, filter, debounce } from 'rxjs/operators'; +import { View, Runtime, Spec } from 'vega'; +import { i18n } from '@kbn/i18n'; +import { Assign } from '@kbn/utility-types'; + +interface DebugValues { + view: View; + spec: Spec; +} + +export interface VegaRuntimeData { + columns: Array<{ + id: string; + }>; + data: Array>; +} + +export type InspectDataSets = Assign; +export type InspectSignalsSets = VegaRuntimeData; + +const vegaAdapterSignalLabel = i18n.translate('visTypeVega.inspector.vegaAdapter.signal', { + defaultMessage: 'Signal', +}); + +const vegaAdapterValueLabel = i18n.translate('visTypeVega.inspector.vegaAdapter.value', { + defaultMessage: 'Value', +}); + +/** Get Runtime Scope for Vega View + * @link https://vega.github.io/vega/docs/api/debugging/#scope + **/ +const getVegaRuntimeScope = (debugValues: DebugValues) => + (debugValues.view as any)._runtime as Runtime; + +const serializeColumns = (item: Record, columns: string[]) => { + const nonSerializableFieldLabel = '(..)'; + + return columns.reduce((row: Record, column) => { + try { + const cell = item[column]; + row[column] = typeof cell === 'object' ? JSON.stringify(cell) : `${cell}`; + } catch (e) { + row[column] = nonSerializableFieldLabel; + } + return row; + }, {}); +}; + +export class VegaAdapter { + private debugValuesSubject = new ReplaySubject(); + + bindInspectValues(debugValues: DebugValues) { + this.debugValuesSubject.next(debugValues); + } + + getDataSetsSubscription(): Observable { + return this.debugValuesSubject.pipe( + filter((debugValues) => Boolean(debugValues)), + map((debugValues) => { + const runtimeScope = getVegaRuntimeScope(debugValues); + + return Object.keys(runtimeScope.data || []).reduce((acc: InspectDataSets[], key) => { + const value = runtimeScope.data[key].values.value; + + if (value && value[0]) { + const columns = Object.keys(value[0]); + acc.push({ + id: key, + columns: columns.map((column) => ({ id: column, schema: 'json' })), + data: value.map((item: Record) => serializeColumns(item, columns)), + }); + } + return acc; + }, []); + }) + ); + } + + getSignalsSetsSubscription(): Observable { + const signalsListener = this.debugValuesSubject.pipe( + filter((debugValues) => Boolean(debugValues)), + switchMap((debugValues) => { + const runtimeScope = getVegaRuntimeScope(debugValues); + + return merge( + ...Object.keys(runtimeScope.signals).map((key: string) => + fromEventPattern( + (handler) => debugValues.view.addSignalListener(key, handler), + (handler) => debugValues.view.removeSignalListener(key, handler) + ) + ) + ).pipe( + debounce((val) => timer(350)), + map(() => debugValues) + ); + }) + ); + + return merge(this.debugValuesSubject, signalsListener).pipe( + filter((debugValues) => Boolean(debugValues)), + map((debugValues) => { + const runtimeScope = getVegaRuntimeScope(debugValues); + + return { + columns: [ + { id: vegaAdapterSignalLabel, schema: 'text' }, + { id: vegaAdapterValueLabel, schema: 'json' }, + ], + data: Object.keys(runtimeScope.signals).map((key: string) => + serializeColumns( + { + [vegaAdapterSignalLabel]: key, + [vegaAdapterValueLabel]: runtimeScope.signals[key].value, + }, + [vegaAdapterSignalLabel, vegaAdapterValueLabel] + ) + ), + }; + }) + ); + } + + getSpecSubscription(): Observable { + return this.debugValuesSubject.pipe( + filter((debugValues) => Boolean(debugValues)), + map((debugValues) => JSON.stringify(debugValues.spec, null, 2)) + ); + } +} diff --git a/src/plugins/vis_type_vega/public/vega_inspector/vega_data_inspector.scss b/src/plugins/vis_type_vega/public/vega_inspector/vega_data_inspector.scss new file mode 100644 index 0000000000000..487f505657d3b --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/vega_data_inspector.scss @@ -0,0 +1,18 @@ +.vgaVegaDataInspector, +.vgaVegaDataInspector__specViewer { + height: 100%; +} + +.vgaVegaDataInspector { + // TODO: EUI needs to provide props to pass down from EuiTabbedContent to tabs and content + display: flex; + flex-direction: column; + + [role='tablist'] { + flex-shrink: 0; + } + + [role='tabpanel'] { + flex-grow: 1; + } +} diff --git a/src/plugins/vis_type_vega/public/vega_inspector/vega_data_inspector.tsx b/src/plugins/vis_type_vega/public/vega_inspector/vega_data_inspector.tsx new file mode 100644 index 0000000000000..3b9427c96e62a --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/vega_data_inspector.tsx @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import './vega_data_inspector.scss'; + +import React from 'react'; +import { EuiTabbedContent } from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; +import { VegaInspectorAdapters } from './vega_inspector'; +import { DataViewer, SignalViewer, SpecViewer } from './components'; +import { InspectorViewProps } from '../../../inspector/public'; + +export type VegaDataInspectorProps = InspectorViewProps; + +const dataSetsLabel = i18n.translate('visTypeVega.inspector.dataSetsLabel', { + defaultMessage: 'Data sets', +}); + +const signalValuesLabel = i18n.translate('visTypeVega.inspector.signalValuesLabel', { + defaultMessage: 'Signal values', +}); + +const specLabel = i18n.translate('visTypeVega.inspector.specLabel', { + defaultMessage: 'Spec', +}); + +export const VegaDataInspector = ({ adapters }: VegaDataInspectorProps) => { + const tabs = [ + { + id: 'data-viewer--id', + name: dataSetsLabel, + content: , + }, + { + id: 'signal-viewer--id', + name: signalValuesLabel, + content: , + }, + { + id: 'spec-viewer--id', + name: specLabel, + content: ( + + ), + }, + ]; + + return ( + + ); +}; diff --git a/src/plugins/vis_type_vega/public/vega_inspector/vega_inspector.tsx b/src/plugins/vis_type_vega/public/vega_inspector/vega_inspector.tsx new file mode 100644 index 0000000000000..83d9e467646a6 --- /dev/null +++ b/src/plugins/vis_type_vega/public/vega_inspector/vega_inspector.tsx @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; + +import { i18n } from '@kbn/i18n'; +import { IUiSettingsClient } from 'kibana/public'; +import { VegaAdapter } from './vega_adapter'; +import { VegaDataInspector, VegaDataInspectorProps } from './vega_data_inspector'; +import { KibanaContextProvider } from '../../../kibana_react/public'; +import { Adapters, RequestAdapter, InspectorViewDescription } from '../../../inspector/public'; + +export interface VegaInspectorAdapters extends Adapters { + requests: RequestAdapter; + vega: VegaAdapter; +} + +const vegaDebugLabel = i18n.translate('visTypeVega.inspector.vegaDebugLabel', { + defaultMessage: 'Vega debug', +}); + +interface VegaInspectorViewDependencies { + uiSettings: IUiSettingsClient; +} + +export const getVegaInspectorView = (dependencies: VegaInspectorViewDependencies) => + ({ + title: vegaDebugLabel, + shouldShow(adapters) { + return Boolean(adapters.vega); + }, + component: (props) => ( + + + + ), + } as InspectorViewDescription); + +export const createInspectorAdapters = (): VegaInspectorAdapters => ({ + requests: new RequestAdapter(), + vega: new VegaAdapter(), +}); diff --git a/src/plugins/vis_type_vega/public/vega_request_handler.ts b/src/plugins/vis_type_vega/public/vega_request_handler.ts index 997b1982d749a..c09a9466df602 100644 --- a/src/plugins/vis_type_vega/public/vega_request_handler.ts +++ b/src/plugins/vis_type_vega/public/vega_request_handler.ts @@ -25,6 +25,7 @@ import { TimeCache } from './data_model/time_cache'; import { VegaVisualizationDependencies } from './plugin'; import { VisParams } from './vega_fn'; import { getData, getInjectedMetadata } from './services'; +import { VegaInspectorAdapters } from './vega_inspector'; interface VegaRequestHandlerParams { query: Query; @@ -33,9 +34,14 @@ interface VegaRequestHandlerParams { visParams: VisParams; } +interface VegaRequestHandlerContext { + abortSignal?: AbortSignal; + inspectorAdapters?: VegaInspectorAdapters; +} + export function createVegaRequestHandler( { plugins: { data }, core: { uiSettings }, serviceSettings }: VegaVisualizationDependencies, - abortSignal?: AbortSignal + context: VegaRequestHandlerContext = {} ) { let searchAPI: SearchAPI; const { timefilter } = data.query.timefilter; @@ -54,7 +60,8 @@ export function createVegaRequestHandler( search: getData().search, injectedMetadata: getInjectedMetadata(), }, - abortSignal + context.abortSignal, + context.inspectorAdapters ); } diff --git a/src/plugins/vis_type_vega/public/vega_type.ts b/src/plugins/vis_type_vega/public/vega_type.ts index 5825661f9001c..d69eb3cfba282 100644 --- a/src/plugins/vis_type_vega/public/vega_type.ts +++ b/src/plugins/vis_type_vega/public/vega_type.ts @@ -23,9 +23,10 @@ import { VegaVisualizationDependencies } from './plugin'; import { VegaVisEditor } from './components'; import { createVegaRequestHandler } from './vega_request_handler'; -// @ts-ignore +// @ts-expect-error import { createVegaVisualization } from './vega_visualization'; import { getDefaultSpec } from './default_spec'; +import { createInspectorAdapters } from './vega_inspector'; export const createVegaTypeDefinition = (dependencies: VegaVisualizationDependencies) => { const requestHandler = createVegaRequestHandler(dependencies); @@ -54,5 +55,6 @@ export const createVegaTypeDefinition = (dependencies: VegaVisualizationDependen showFilterBar: true, }, stage: 'experimental', + inspectorAdapters: createInspectorAdapters, }; }; diff --git a/src/plugins/vis_type_vega/public/vega_view/vega_base_view.js b/src/plugins/vis_type_vega/public/vega_view/vega_base_view.js index 55c3606bf5e45..8f88d5c5b2056 100644 --- a/src/plugins/vis_type_vega/public/vega_view/vega_base_view.js +++ b/src/plugins/vis_type_vega/public/vega_view/vega_base_view.js @@ -364,6 +364,11 @@ export class VegaBaseView { * Set global debug variable to simplify vega debugging in console. Show info message first time */ setDebugValues(view, spec, vlspec) { + this._parser.searchAPI.inspectorAdapters?.vega.bindInspectValues({ + view, + spec: vlspec || spec, + }); + if (window) { if (window.VEGA_DEBUG === undefined && console) { console.log('%cWelcome to Kibana Vega Plugin!', 'font-size: 16px; font-weight: bold;'); diff --git a/src/plugins/vis_type_vega/public/vega_view/vega_map_view.js b/src/plugins/vis_type_vega/public/vega_view/vega_map_view.js index 6908fd13a9ca1..78ae2efdbdda5 100644 --- a/src/plugins/vis_type_vega/public/vega_view/vega_map_view.js +++ b/src/plugins/vis_type_vega/public/vega_view/vega_map_view.js @@ -142,7 +142,7 @@ export class VegaMapView extends VegaBaseView { }); const vegaView = vegaMapLayer.getVegaView(); - this.setDebugValues(vegaView, this._parser.spec, this._parser.vlspec); await this.setView(vegaView); + this.setDebugValues(vegaView, this._parser.spec, this._parser.vlspec); } } diff --git a/src/plugins/vis_type_vega/public/vega_view/vega_view.js b/src/plugins/vis_type_vega/public/vega_view/vega_view.js index e3455b97b7fe2..98c972ef84ccb 100644 --- a/src/plugins/vis_type_vega/public/vega_view/vega_view.js +++ b/src/plugins/vis_type_vega/public/vega_view/vega_view.js @@ -26,7 +26,6 @@ export class VegaView extends VegaBaseView { if (!this._$container) return; const view = new vega.View(vega.parse(this._parser.spec), this._vegaViewConfig); - this.setDebugValues(view, this._parser.spec, this._parser.vlspec); view.warn = this.onWarn.bind(this); view.error = this.onError.bind(this); @@ -36,5 +35,6 @@ export class VegaView extends VegaBaseView { if (this._parser.useHover) view.hover(); await this.setView(view); + this.setDebugValues(view, this._parser.spec, this._parser.vlspec); } } diff --git a/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts b/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts index 2f9cda32fccdc..749926e1abd00 100644 --- a/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts +++ b/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts @@ -34,6 +34,7 @@ import { EmbeddableOutput, Embeddable, IContainer, + Adapters, } from '../../../../plugins/embeddable/public'; import { dispatchRenderComplete } from '../../../../plugins/kibana_utils/public'; import { @@ -78,8 +79,6 @@ export interface VisualizeOutput extends EmbeddableOutput { type ExpressionLoader = InstanceType; -const visTypesWithoutInspector = ['markdown', 'input_control_vis', 'metrics', 'vega', 'timelion']; - export class VisualizeEmbeddable extends Embeddable { private handler?: ExpressionLoader; private timefilter: TimefilterContract; @@ -96,6 +95,7 @@ export class VisualizeEmbeddable extends Embeddable { - if (!this.handler || visTypesWithoutInspector.includes(this.vis.type.name)) { + if (!this.handler || (this.inspectorAdapters && !Object.keys(this.inspectorAdapters).length)) { return undefined; } return this.handler.inspect(); @@ -349,6 +356,7 @@ export class VisualizeEmbeddable extends Embeddable Adapters); } export class BaseVisType { @@ -63,6 +65,7 @@ export class BaseVisType { hierarchicalData: boolean | unknown; setup?: unknown; useCustomNoDataScreen: boolean; + inspectorAdapters?: Adapters | (() => Adapters); constructor(opts: BaseVisTypeOptions) { if (!opts.icon && !opts.image) { @@ -98,6 +101,7 @@ export class BaseVisType { this.requiresSearch = this.requestHandler !== 'none'; this.hierarchicalData = opts.hierarchicalData || false; this.useCustomNoDataScreen = opts.useCustomNoDataScreen || false; + this.inspectorAdapters = opts.inspectorAdapters; } public get schemas() { diff --git a/test/functional/apps/visualize/_vega_chart.js b/test/functional/apps/visualize/_vega_chart.js index 4442e1f969b4b..c530c6f823133 100644 --- a/test/functional/apps/visualize/_vega_chart.js +++ b/test/functional/apps/visualize/_vega_chart.js @@ -22,7 +22,6 @@ import expect from '@kbn/expect'; export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['timePicker', 'visualize', 'visChart', 'vegaChart']); const filterBar = getService('filterBar'); - const inspector = getService('inspector'); const log = getService('log'); describe('vega chart in visualize app', () => { @@ -35,10 +34,6 @@ export default function ({ getService, getPageObjects }) { describe('vega chart', () => { describe('initial render', () => { - it('should not have inspector enabled', async function () { - await inspector.expectIsNotEnabled(); - }); - it.skip('should have some initial vega spec text', async function () { const vegaSpec = await PageObjects.vegaChart.getSpec(); expect(vegaSpec).to.contain('{').and.to.contain('data'); diff --git a/x-pack/plugins/maps/public/kibana_services.js b/x-pack/plugins/maps/public/kibana_services.js index 53e128f94dfb6..89d578f27b118 100644 --- a/x-pack/plugins/maps/public/kibana_services.js +++ b/x-pack/plugins/maps/public/kibana_services.js @@ -80,7 +80,7 @@ export async function fetchSearchSourceAndRecordWithInspector({ inspectorRequest.json(body); }); resp = await searchSource.fetch({ abortSignal }); - inspectorRequest.stats(getResponseInspectorStats(searchSource, resp)).ok({ json: resp }); + inspectorRequest.stats(getResponseInspectorStats(resp, searchSource)).ok({ json: resp }); } catch (error) { inspectorRequest.error({ error }); throw error; From c74b214fe3815e4ebe7e12cfc804651033fd2fa2 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Tue, 21 Jul 2020 08:14:12 -0400 Subject: [PATCH 005/104] allow some env settings for ingest manager (#72544) --- .../os_packages/docker_generator/resources/bin/kibana-docker | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker index 745a3d1f0c830..0913d4ba4e83a 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker @@ -174,6 +174,8 @@ kibana_vars=( xpack.infra.sources.default.fields.timestamp xpack.infra.sources.default.logAlias xpack.infra.sources.default.metricAlias + xpack.ingestManager.fleet.tlsCheckDisabled + xpack.ingestManager.registryUrl xpack.license_management.enabled xpack.ml.enabled xpack.reporting.capture.browser.autoDownload From 8fdebc9e822af030aa949554d00c69f3143f41ed Mon Sep 17 00:00:00 2001 From: Gidi Meir Morris Date: Tue, 21 Jul 2020 14:08:29 +0100 Subject: [PATCH 006/104] [Task Manager] Batches the update operations in Task Manager (#71470) This PR attempts to batch update tasks in Task Manager in order to avoid overloading the Elasticsearch queue. This is the 1st PR addressing https://github.com/elastic/kibana/issues/65551 Under the hood we now use a Reactive buffer accumulates all calls to the `update` api in the TaskStore and flushes after 50ms or when as many operations as there are workers have been buffered (whichever comes first). --- .../server/buffered_task_store.test.ts | 82 +++++ .../server/buffered_task_store.ts | 39 +++ .../server/lib/bulk_operation_buffer.test.ts | 288 ++++++++++++++++++ .../server/lib/bulk_operation_buffer.ts | 129 ++++++++ .../server/lib/result_type.test.ts | 27 ++ .../task_manager/server/lib/result_type.ts | 19 ++ .../task_manager/server/task_manager.ts | 10 +- .../task_manager/server/task_runner.ts | 2 +- .../task_manager/server/task_store.mock.ts | 31 ++ .../plugins/task_manager/server/task_store.ts | 65 +++- 10 files changed, 685 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/task_manager/server/buffered_task_store.test.ts create mode 100644 x-pack/plugins/task_manager/server/buffered_task_store.ts create mode 100644 x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.test.ts create mode 100644 x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts create mode 100644 x-pack/plugins/task_manager/server/lib/result_type.test.ts create mode 100644 x-pack/plugins/task_manager/server/task_store.mock.ts diff --git a/x-pack/plugins/task_manager/server/buffered_task_store.test.ts b/x-pack/plugins/task_manager/server/buffered_task_store.test.ts new file mode 100644 index 0000000000000..8e18405c79ed2 --- /dev/null +++ b/x-pack/plugins/task_manager/server/buffered_task_store.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import { taskStoreMock } from './task_store.mock'; +import { BufferedTaskStore } from './buffered_task_store'; +import { asErr, asOk } from './lib/result_type'; +import { TaskStatus } from './task'; + +describe('Buffered Task Store', () => { + test('proxies the TaskStore for `maxAttempts` and `remove`', async () => { + const taskStore = taskStoreMock.create({ maxAttempts: 10 }); + taskStore.bulkUpdate.mockResolvedValue([]); + const bufferedStore = new BufferedTaskStore(taskStore, {}); + + expect(bufferedStore.maxAttempts).toEqual(10); + + bufferedStore.remove('1'); + expect(taskStore.remove).toHaveBeenCalledWith('1'); + }); + + describe('update', () => { + test("proxies the TaskStore's `bulkUpdate`", async () => { + const taskStore = taskStoreMock.create({ maxAttempts: 10 }); + const bufferedStore = new BufferedTaskStore(taskStore, {}); + + const task = mockTask(); + + taskStore.bulkUpdate.mockResolvedValue([asOk(task)]); + + expect(await bufferedStore.update(task)).toMatchObject(task); + expect(taskStore.bulkUpdate).toHaveBeenCalledWith([task]); + }); + + test('handles partially successfull bulkUpdates resolving each call appropriately', async () => { + const taskStore = taskStoreMock.create({ maxAttempts: 10 }); + const bufferedStore = new BufferedTaskStore(taskStore, {}); + + const tasks = [mockTask(), mockTask(), mockTask()]; + + taskStore.bulkUpdate.mockResolvedValueOnce([ + asOk(tasks[0]), + asErr({ entity: tasks[1], error: new Error('Oh no, something went terribly wrong') }), + asOk(tasks[2]), + ]); + + const results = [ + bufferedStore.update(tasks[0]), + bufferedStore.update(tasks[1]), + bufferedStore.update(tasks[2]), + ]; + expect(await results[0]).toMatchObject(tasks[0]); + expect(results[1]).rejects.toMatchInlineSnapshot( + `[Error: Oh no, something went terribly wrong]` + ); + expect(await results[2]).toMatchObject(tasks[2]); + }); + }); +}); + +function mockTask() { + return { + id: `task_${uuid.v4()}`, + attempts: 0, + schedule: undefined, + params: { hello: 'world' }, + retryAt: null, + runAt: new Date(), + scheduledAt: new Date(), + scope: undefined, + startedAt: null, + state: { foo: 'bar' }, + status: TaskStatus.Idle, + taskType: 'report', + user: undefined, + version: '123', + ownerId: '123', + }; +} diff --git a/x-pack/plugins/task_manager/server/buffered_task_store.ts b/x-pack/plugins/task_manager/server/buffered_task_store.ts new file mode 100644 index 0000000000000..e1e5f802204c1 --- /dev/null +++ b/x-pack/plugins/task_manager/server/buffered_task_store.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TaskStore } from './task_store'; +import { ConcreteTaskInstance } from './task'; +import { Updatable } from './task_runner'; +import { createBuffer, Operation, BufferOptions } from './lib/bulk_operation_buffer'; +import { unwrapPromise } from './lib/result_type'; + +// by default allow updates to be buffered for up to 50ms +const DEFAULT_BUFFER_MAX_DURATION = 50; + +export class BufferedTaskStore implements Updatable { + private bufferedUpdate: Operation; + constructor(private readonly taskStore: TaskStore, options: BufferOptions) { + this.bufferedUpdate = createBuffer( + (docs) => taskStore.bulkUpdate(docs), + { + bufferMaxDuration: DEFAULT_BUFFER_MAX_DURATION, + ...options, + } + ); + } + + public get maxAttempts(): number { + return this.taskStore.maxAttempts; + } + + public async update(doc: ConcreteTaskInstance): Promise { + return unwrapPromise(this.bufferedUpdate(doc)); + } + + public async remove(id: string): Promise { + return this.taskStore.remove(id); + } +} diff --git a/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.test.ts b/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.test.ts new file mode 100644 index 0000000000000..9293656233026 --- /dev/null +++ b/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.test.ts @@ -0,0 +1,288 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createBuffer, Entity, OperationError, BulkOperation } from './bulk_operation_buffer'; +import { mapErr, asOk, asErr, Ok, Err } from './result_type'; + +interface TaskInstance extends Entity { + attempts: number; +} + +const createTask = (function (): () => TaskInstance { + let counter = 0; + return () => ({ + id: `task ${++counter}`, + attempts: 1, + }); +})(); + +function incrementAttempts(task: TaskInstance): Ok { + return asOk({ + ...task, + attempts: task.attempts + 1, + }); +} + +function errorAttempts(task: TaskInstance): Err> { + return asErr({ + entity: incrementAttempts(task).value, + error: { name: '', message: 'Oh no, something went terribly wrong', statusCode: 500 }, + }); +} + +describe('Bulk Operation Buffer', () => { + describe('createBuffer()', () => { + test('batches up multiple Operation calls', async () => { + const bulkUpdate: jest.Mocked> = jest.fn( + ([task1, task2]) => { + return Promise.resolve([incrementAttempts(task1), incrementAttempts(task2)]); + } + ); + + const bufferedUpdate = createBuffer(bulkUpdate); + + const task1 = createTask(); + const task2 = createTask(); + + expect(await Promise.all([bufferedUpdate(task1), bufferedUpdate(task2)])).toMatchObject([ + incrementAttempts(task1), + incrementAttempts(task2), + ]); + expect(bulkUpdate).toHaveBeenCalledWith([task1, task2]); + }); + + test('batch updates are executed at most by the next Event Loop tick by default', async () => { + const bulkUpdate: jest.Mocked> = jest.fn((tasks) => { + return Promise.resolve(tasks.map(incrementAttempts)); + }); + + const bufferedUpdate = createBuffer(bulkUpdate); + + const task1 = createTask(); + const task2 = createTask(); + const task3 = createTask(); + const task4 = createTask(); + const task5 = createTask(); + const task6 = createTask(); + + return new Promise((resolve) => { + Promise.all([bufferedUpdate(task1), bufferedUpdate(task2)]).then((_) => { + expect(bulkUpdate).toHaveBeenCalledTimes(1); + expect(bulkUpdate).toHaveBeenCalledWith([task1, task2]); + expect(bulkUpdate).not.toHaveBeenCalledWith([task3, task4]); + }); + + setTimeout(() => { + // on next tick + setTimeout(() => { + // on next tick + expect(bulkUpdate).toHaveBeenCalledTimes(2); + Promise.all([bufferedUpdate(task5), bufferedUpdate(task6)]).then((_) => { + expect(bulkUpdate).toHaveBeenCalledTimes(3); + expect(bulkUpdate).toHaveBeenCalledWith([task5, task6]); + resolve(); + }); + }, 0); + + expect(bulkUpdate).toHaveBeenCalledTimes(1); + Promise.all([bufferedUpdate(task3), bufferedUpdate(task4)]).then((_) => { + expect(bulkUpdate).toHaveBeenCalledTimes(2); + expect(bulkUpdate).toHaveBeenCalledWith([task3, task4]); + }); + }, 0); + }); + }); + + test('batch updates can be customised to execute after a certain period', async () => { + const bulkUpdate: jest.Mocked> = jest.fn((tasks) => { + return Promise.resolve(tasks.map(incrementAttempts)); + }); + + const bufferMaxDuration = 50; + const bufferedUpdate = createBuffer(bulkUpdate, { bufferMaxDuration }); + + const task1 = createTask(); + const task2 = createTask(); + const task3 = createTask(); + const task4 = createTask(); + const task5 = createTask(); + const task6 = createTask(); + + return new Promise((resolve) => { + Promise.all([bufferedUpdate(task1), bufferedUpdate(task2)]).then((_) => { + expect(bulkUpdate).toHaveBeenCalledTimes(1); + expect(bulkUpdate).toHaveBeenCalledWith([task1, task2]); + expect(bulkUpdate).not.toHaveBeenCalledWith([task3, task4]); + }); + + setTimeout(() => { + // on next tick + setTimeout(() => { + // on next tick + expect(bulkUpdate).toHaveBeenCalledTimes(2); + Promise.all([bufferedUpdate(task5), bufferedUpdate(task6)]).then((_) => { + expect(bulkUpdate).toHaveBeenCalledTimes(3); + expect(bulkUpdate).toHaveBeenCalledWith([task5, task6]); + resolve(); + }); + }, bufferMaxDuration + 1); + + expect(bulkUpdate).toHaveBeenCalledTimes(1); + Promise.all([bufferedUpdate(task3), bufferedUpdate(task4)]).then((_) => { + expect(bulkUpdate).toHaveBeenCalledTimes(2); + expect(bulkUpdate).toHaveBeenCalledWith([task3, task4]); + }); + }, bufferMaxDuration + 1); + }); + }); + + test('batch updates are executed once queue hits a certain bound', async () => { + const bulkUpdate: jest.Mocked> = jest.fn((tasks) => { + return Promise.resolve(tasks.map(incrementAttempts)); + }); + + const bufferedUpdate = createBuffer(bulkUpdate, { + bufferMaxDuration: 100, + bufferMaxOperations: 2, + }); + + const task1 = createTask(); + const task2 = createTask(); + const task3 = createTask(); + const task4 = createTask(); + const task5 = createTask(); + + return new Promise((resolve) => { + bufferedUpdate(task1); + bufferedUpdate(task2); + bufferedUpdate(task3); + bufferedUpdate(task4); + + setTimeout(() => { + expect(bulkUpdate).toHaveBeenCalledTimes(2); + expect(bulkUpdate).toHaveBeenCalledWith([task1, task2]); + expect(bulkUpdate).toHaveBeenCalledWith([task3, task4]); + + setTimeout(() => { + expect(bulkUpdate).toHaveBeenCalledTimes(2); + bufferedUpdate(task5).then((_) => { + expect(bulkUpdate).toHaveBeenCalledTimes(3); + expect(bulkUpdate).toHaveBeenCalledWith([task5]); + resolve(); + }); + }, 50); + }, 50); + }); + }); + + test('queue upper bound is reset after each flush', async () => { + const bulkUpdate: jest.Mocked> = jest.fn((tasks) => { + return Promise.resolve(tasks.map(incrementAttempts)); + }); + + const bufferMaxDuration = 100; + const bufferedUpdate = createBuffer(bulkUpdate, { + bufferMaxDuration, + bufferMaxOperations: 3, + }); + + const task1 = createTask(); + const task2 = createTask(); + const task3 = createTask(); + const task4 = createTask(); + + return new Promise((resolve) => { + bufferedUpdate(task1); + bufferedUpdate(task2); + + setTimeout(() => { + expect(bulkUpdate).toHaveBeenCalledTimes(1); + expect(bulkUpdate).toHaveBeenCalledWith([task1, task2]); + + bufferedUpdate(task3); + bufferedUpdate(task4); + + setTimeout(() => { + expect(bulkUpdate).toHaveBeenCalledTimes(1); + + setTimeout(() => { + expect(bulkUpdate).toHaveBeenCalledTimes(2); + expect(bulkUpdate).toHaveBeenCalledWith([task3, task4]); + resolve(); + }, bufferMaxDuration / 2); + }, bufferMaxDuration / 2); + }, bufferMaxDuration + 1); + }); + }); + test('handles both resolutions and rejections at individual task level', async (done) => { + const bulkUpdate: jest.Mocked> = jest.fn( + ([task1, task2, task3]) => { + return Promise.resolve([ + incrementAttempts(task1), + errorAttempts(task2), + incrementAttempts(task3), + ]); + } + ); + + const bufferedUpdate = createBuffer(bulkUpdate); + + const task1 = createTask(); + const task2 = createTask(); + const task3 = createTask(); + + return Promise.all([ + expect(bufferedUpdate(task1)).resolves.toMatchObject(incrementAttempts(task1)), + expect(bufferedUpdate(task2)).rejects.toMatchObject( + mapErr( + (err: OperationError) => asErr(err.error), + errorAttempts(task2) + ) + ), + expect(bufferedUpdate(task3)).resolves.toMatchObject(incrementAttempts(task3)), + ]).then(() => { + expect(bulkUpdate).toHaveBeenCalledTimes(1); + done(); + }); + }); + + test('handles bulkUpdate failure', async (done) => { + const bulkUpdate: jest.Mocked> = jest.fn(() => { + return Promise.reject(new Error('bulkUpdate is an illusion')); + }); + + const bufferedUpdate = createBuffer(bulkUpdate); + + const task1 = createTask(); + const task2 = createTask(); + const task3 = createTask(); + + return Promise.all([ + expect(bufferedUpdate(task1)).rejects.toMatchInlineSnapshot(` + Object { + "error": [Error: bulkUpdate is an illusion], + "tag": "err", + } + `), + expect(bufferedUpdate(task2)).rejects.toMatchInlineSnapshot(` + Object { + "error": [Error: bulkUpdate is an illusion], + "tag": "err", + } + `), + expect(bufferedUpdate(task3)).rejects.toMatchInlineSnapshot(` + Object { + "error": [Error: bulkUpdate is an illusion], + "tag": "err", + } + `), + ]).then(() => { + expect(bulkUpdate).toHaveBeenCalledTimes(1); + done(); + }); + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts b/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts new file mode 100644 index 0000000000000..fca7ce02e0cd7 --- /dev/null +++ b/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { keyBy, map } from 'lodash'; +import { Subject, race, from } from 'rxjs'; +import { bufferWhen, filter, bufferCount, flatMap, mapTo, first } from 'rxjs/operators'; +import { either, Result, asOk, asErr, Ok, Err } from './result_type'; + +export interface BufferOptions { + bufferMaxDuration?: number; + bufferMaxOperations?: number; +} + +export interface Entity { + id: string; +} + +export interface OperationError { + entity: Input; + error: ErrorOutput; +} + +export type OperationResult = Result< + Output, + OperationError +>; + +export type Operation = ( + entity: Input +) => Promise>; + +export type BulkOperation = ( + entities: Input[] +) => Promise>>; + +const DONT_FLUSH = false; +const FLUSH = true; + +export function createBuffer( + bulkOperation: BulkOperation, + { bufferMaxDuration = 0, bufferMaxOperations = Number.MAX_VALUE }: BufferOptions = {} +): Operation { + const flushBuffer = new Subject(); + + const storeUpdateBuffer = new Subject<{ + entity: Input; + onSuccess: (entity: Ok) => void; + onFailure: (error: Err) => void; + }>(); + + storeUpdateBuffer + .pipe( + bufferWhen(() => flushBuffer), + filter((tasks) => tasks.length > 0) + ) + .subscribe((entities) => { + const entityById = keyBy(entities, ({ entity: { id } }) => id); + bulkOperation(map(entities, 'entity')) + .then((results) => { + results.forEach((result) => + either( + result, + (entity) => { + entityById[entity.id].onSuccess(asOk(entity)); + }, + ({ entity, error }: OperationError) => { + entityById[entity.id].onFailure(asErr(error)); + } + ) + ); + }) + .catch((ex) => { + entities.forEach(({ onFailure }) => onFailure(asErr(ex))); + }); + }); + + let countInBuffer = 0; + const flushAndResetCounter = () => { + countInBuffer = 0; + flushBuffer.next(); + }; + storeUpdateBuffer + .pipe( + // complete once the buffer has either filled to `bufferMaxOperations` or + // a `bufferMaxDuration` has passed. Default to `bufferMaxDuration` being the + // current event loop tick rather than a fixed duration + flatMap(() => { + return ++countInBuffer === 1 + ? race([ + // the race is started in response to the first operation into the buffer + // so we flush once the remaining operations come in (which is `bufferMaxOperations - 1`) + storeUpdateBuffer.pipe(bufferCount(bufferMaxOperations - 1)), + bufferMaxDuration + ? // if theres a max duration, flush buffer based on that + from(resolveIn(bufferMaxDuration)) + : // ensure we flush by the end of the "current" event loop tick + from(resolveImmediate()), + ]).pipe(first(), mapTo(FLUSH)) + : from([DONT_FLUSH]); + }), + filter((shouldFlush) => shouldFlush) + ) + .subscribe({ + next: flushAndResetCounter, + // As this stream is just trying to decide when to flush + // there's no data to lose, so in the case that an error + // is thrown, lets just flush + error: flushAndResetCounter, + }); + + return async function (entity: Input) { + return new Promise((resolve, reject) => { + storeUpdateBuffer.next({ entity, onSuccess: resolve, onFailure: reject }); + }); + }; +} + +function resolveImmediate() { + return new Promise(setImmediate); +} + +function resolveIn(ms: number) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} diff --git a/x-pack/plugins/task_manager/server/lib/result_type.test.ts b/x-pack/plugins/task_manager/server/lib/result_type.test.ts new file mode 100644 index 0000000000000..480a732f1f617 --- /dev/null +++ b/x-pack/plugins/task_manager/server/lib/result_type.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import { unwrapPromise, asOk, asErr } from './result_type'; + +describe(`Result`, () => { + describe(`unwrapPromise`, () => { + test(`unwraps OKs from the result`, async () => { + const uniqueId = uuid.v4(); + expect(await unwrapPromise(Promise.resolve(asOk(uniqueId)))).toEqual(uniqueId); + }); + + test(`unwraps Errs from the result`, async () => { + const uniqueId = uuid.v4(); + expect(unwrapPromise(Promise.resolve(asErr(uniqueId)))).rejects.toEqual(uniqueId); + }); + + test(`unwraps Errs from the result when promise rejects`, async () => { + const uniqueId = uuid.v4(); + expect(unwrapPromise(Promise.reject(asErr(uniqueId)))).rejects.toEqual(uniqueId); + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/lib/result_type.ts b/x-pack/plugins/task_manager/server/lib/result_type.ts index edf4d84dd226d..d21c17d3bb5b3 100644 --- a/x-pack/plugins/task_manager/server/lib/result_type.ts +++ b/x-pack/plugins/task_manager/server/lib/result_type.ts @@ -47,6 +47,25 @@ export async function promiseResult(future: Promise): Promise(future: Promise>): Promise { + return future + .catch( + // catch rejection as we expect the result of the rejected promise + // to be wrapped in a Result - sadly there's no way to "Type" this + // requirment in Typescript as Promises do not enfore a type on their + // rejection + // The `then` will then unwrap the Result from around `ex` for us + (ex: Err) => ex + ) + .then((result: Result) => + map( + result, + (value: T) => Promise.resolve(value), + (err: E) => Promise.reject(err) + ) + ); +} + export function unwrap(result: Result): T | E { return isOk(result) ? result.value : result.error; } diff --git a/x-pack/plugins/task_manager/server/task_manager.ts b/x-pack/plugins/task_manager/server/task_manager.ts index 23cb33cfac6c2..35ca439bb9130 100644 --- a/x-pack/plugins/task_manager/server/task_manager.ts +++ b/x-pack/plugins/task_manager/server/task_manager.ts @@ -57,6 +57,7 @@ import { } from './task_store'; import { identifyEsError } from './lib/identify_es_error'; import { ensureDeprecatedFieldsAreCorrected } from './lib/correct_deprecated_fields'; +import { BufferedTaskStore } from './buffered_task_store'; const VERSION_CONFLICT_STATUS = 409; @@ -90,7 +91,10 @@ export type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim | TaskRun */ export class TaskManager { private definitions: TaskDictionary = {}; + private store: TaskStore; + private bufferedStore: BufferedTaskStore; + private logger: Logger; private pool: TaskPool; // all task related events (task claimed, task marked as running, etc.) are emitted through events$ @@ -139,6 +143,10 @@ export class TaskManager { // pipe store events into the TaskManager's event stream this.store.events.subscribe((event) => this.events$.next(event)); + this.bufferedStore = new BufferedTaskStore(this.store, { + bufferMaxOperations: opts.config.max_workers, + }); + this.pool = new TaskPool({ logger: this.logger, maxWorkers: opts.config.max_workers, @@ -165,7 +173,7 @@ export class TaskManager { return new TaskManagerRunner({ logger: this.logger, instance, - store: this.store, + store: this.bufferedStore, definitions: this.definitions, beforeRun: this.middleware.beforeRun, beforeMarkRunning: this.middleware.beforeMarkRunning, diff --git a/x-pack/plugins/task_manager/server/task_runner.ts b/x-pack/plugins/task_manager/server/task_runner.ts index 4c690a5675f61..ebf13fac2f311 100644 --- a/x-pack/plugins/task_manager/server/task_runner.ts +++ b/x-pack/plugins/task_manager/server/task_runner.ts @@ -49,7 +49,7 @@ export interface TaskRunner { toString: () => string; } -interface Updatable { +export interface Updatable { readonly maxAttempts: number; update(doc: ConcreteTaskInstance): Promise; remove(id: string): Promise; diff --git a/x-pack/plugins/task_manager/server/task_store.mock.ts b/x-pack/plugins/task_manager/server/task_store.mock.ts new file mode 100644 index 0000000000000..86db695bc5e2c --- /dev/null +++ b/x-pack/plugins/task_manager/server/task_store.mock.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TaskStore } from './task_store'; + +interface TaskStoreOptions { + maxAttempts?: number; + index?: string; + taskManagerId?: string; +} +export const taskStoreMock = { + create({ maxAttempts = 0, index = '', taskManagerId = '' }: TaskStoreOptions) { + const mocked = ({ + update: jest.fn(), + remove: jest.fn(), + schedule: jest.fn(), + claimAvailableTasks: jest.fn(), + bulkUpdate: jest.fn(), + get: jest.fn(), + getLifecycle: jest.fn(), + fetch: jest.fn(), + maxAttempts, + index, + taskManagerId, + } as unknown) as jest.Mocked; + return mocked; + }, +}; diff --git a/x-pack/plugins/task_manager/server/task_store.ts b/x-pack/plugins/task_manager/server/task_store.ts index 4a691e17011e8..7ec3db5c99aa7 100644 --- a/x-pack/plugins/task_manager/server/task_store.ts +++ b/x-pack/plugins/task_manager/server/task_store.ts @@ -17,9 +17,10 @@ import { SavedObjectsSerializer, SavedObjectsRawDoc, ISavedObjectsRepository, + SavedObjectsUpdateResponse, } from '../../../../src/core/server'; -import { asOk, asErr } from './lib/result_type'; +import { asOk, asErr, Result } from './lib/result_type'; import { ConcreteTaskInstance, @@ -98,10 +99,10 @@ export interface ClaimOwnershipResult { docs: ConcreteTaskInstance[]; } -export interface BulkUpdateTaskFailureResult { - error: NonNullable; - task: ConcreteTaskInstance; -} +export type BulkUpdateResult = Result< + ConcreteTaskInstance, + { entity: ConcreteTaskInstance; error: Error } +>; export interface UpdateByQueryResult { updated: number; @@ -332,6 +333,54 @@ export class TaskStore { ); } + /** + * Updates the specified docs in the index, returning the docs + * with their versions up to date. + * + * @param {Array} docs + * @returns {Promise>} + */ + public async bulkUpdate(docs: ConcreteTaskInstance[]): Promise { + const attributesByDocId = docs.reduce((attrsById, doc) => { + attrsById.set(doc.id, taskInstanceToAttributes(doc)); + return attrsById; + }, new Map()); + + const updatedSavedObjects: Array = ( + await this.savedObjectsRepository.bulkUpdate( + docs.map((doc) => ({ + type: 'task', + id: doc.id, + options: { version: doc.version }, + attributes: attributesByDocId.get(doc.id)!, + })), + { + refresh: false, + } + ) + ).saved_objects; + + return updatedSavedObjects.map((updatedSavedObject, index) => + isSavedObjectsUpdateResponse(updatedSavedObject) + ? asOk( + savedObjectToConcreteTaskInstance({ + ...updatedSavedObject, + attributes: defaults( + updatedSavedObject.attributes, + attributesByDocId.get(updatedSavedObject.id)! + ), + }) + ) + : asErr({ + // The SavedObjectsRepository maintains the order of the docs + // so we can rely on the index in the `docs` to match an error + // on the same index in the `bulkUpdate` result + entity: docs[index], + error: updatedSavedObject, + }) + ); + } + /** * Removes the specified task from the index. * @@ -468,3 +517,9 @@ function ensureQueryOnlyReturnsTaskObjects(opts: SearchOpts): SearchOpts { query, }; } + +function isSavedObjectsUpdateResponse( + result: SavedObjectsUpdateResponse | Error +): result is SavedObjectsUpdateResponse { + return result && typeof (result as SavedObjectsUpdateResponse).id === 'string'; +} From c63ab91c7bfc574a563a5e0be3bab67156b8ddaa Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Tue, 21 Jul 2020 09:12:50 -0400 Subject: [PATCH 007/104] [Monitoring] Fix the messaging around needing TLS enabled (#72310) * Fix the copy * Fix type issues * PR feedback * Add missing code --- .../public/alerts/lib/security_toasts.tsx | 82 ++----------------- 1 file changed, 5 insertions(+), 77 deletions(-) diff --git a/x-pack/plugins/monitoring/public/alerts/lib/security_toasts.tsx b/x-pack/plugins/monitoring/public/alerts/lib/security_toasts.tsx index 918c0b5c9b609..2850a5b772c32 100644 --- a/x-pack/plugins/monitoring/public/alerts/lib/security_toasts.tsx +++ b/x-pack/plugins/monitoring/public/alerts/lib/security_toasts.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiSpacer, EuiLink, EuiCode, EuiText } from '@elastic/eui'; +import { EuiSpacer, EuiLink } from '@elastic/eui'; import { Legacy } from '../../legacy_shims'; import { toMountPoint } from '../../../../../../src/plugins/kibana_react/public'; @@ -30,11 +30,10 @@ const showTlsAndEncryptionError = () => {

{i18n.translate('xpack.monitoring.healthCheck.tlsAndEncryptionError', { - defaultMessage: `You must enable Transport Layer Security between Kibana and Elasticsearch - and configure an encryption key in your kibana.yml file to use the Alerting feature.`, + defaultMessage: `Stack monitoring alerts require Transport Layer Security between Kibana and Elasticsearch, and an encryption key in your kibana.yml file.`, })}

- + { }); }; -const showEncryptionError = () => { - const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = Legacy.shims.docLinks; - - Legacy.shims.toastNotifications.addWarning( - { - title: toMountPoint( - - ), - text: toMountPoint( -
- {i18n.translate('xpack.monitoring.healthCheck.encryptionErrorBeforeKey', { - defaultMessage: 'To create an alert, set a value for ', - })} - - {'xpack.encryptedSavedObjects.encryptionKey'} - - {i18n.translate('xpack.monitoring.healthCheck.encryptionErrorAfterKey', { - defaultMessage: ' in your kibana.yml file. ', - })} - - {i18n.translate('xpack.monitoring.healthCheck.encryptionErrorAction', { - defaultMessage: 'Learn how.', - })} - -
- ), - }, - {} - ); -}; - -const showTlsError = () => { - const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = Legacy.shims.docLinks; - - Legacy.shims.toastNotifications.addWarning({ - title: toMountPoint( - - ), - text: toMountPoint( -
- {i18n.translate('xpack.monitoring.healthCheck.tlsError', { - defaultMessage: - 'Alerting relies on API keys, which require TLS between Elasticsearch and Kibana. ', - })} - - {i18n.translate('xpack.monitoring.healthCheck.tlsErrorAction', { - defaultMessage: 'Learn how to enable TLS.', - })} - -
- ), - }); -}; - export const showSecurityToast = (alertingHealth: AlertingFrameworkHealth) => { const { isSufficientlySecure, hasPermanentEncryptionKey } = alertingHealth; + if ( Array.isArray(alertingHealth) || (!alertingHealth.hasOwnProperty('isSufficientlySecure') && @@ -127,11 +59,7 @@ export const showSecurityToast = (alertingHealth: AlertingFrameworkHealth) => { return; } - if (!isSufficientlySecure && !hasPermanentEncryptionKey) { + if (!isSufficientlySecure || !hasPermanentEncryptionKey) { showTlsAndEncryptionError(); - } else if (!isSufficientlySecure) { - showTlsError(); - } else if (!hasPermanentEncryptionKey) { - showEncryptionError(); } }; From fbf41e53795abf257caf5b44636d2ad35c4a364c Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 21 Jul 2020 14:28:10 +0100 Subject: [PATCH 008/104] [ML] Handling data recognizer saved object errors (#72447) * [ML] Handling data recognizer saved object errors * adding text for unknown errors * fixing typos --- .../plugins/ml/common/types/capabilities.ts | 2 +- x-pack/plugins/ml/common/types/modules.ts | 1 + .../recognize/components/kibana_objects.tsx | 7 ++++- .../models/data_recognizer/data_recognizer.ts | 30 +++++++++++++++++-- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ml/common/types/capabilities.ts b/x-pack/plugins/ml/common/types/capabilities.ts index b46dd87eec15f..f2177b0a3572f 100644 --- a/x-pack/plugins/ml/common/types/capabilities.ts +++ b/x-pack/plugins/ml/common/types/capabilities.ts @@ -81,7 +81,7 @@ export function getPluginPrivileges() { catalogue: [PLUGIN_ID], savedObject: { all: [], - read: ['index-pattern', 'search'], + read: ['index-pattern', 'dashboard', 'search', 'visualization'], }, }; diff --git a/x-pack/plugins/ml/common/types/modules.ts b/x-pack/plugins/ml/common/types/modules.ts index b476762f6efca..bfa7e38332c1b 100644 --- a/x-pack/plugins/ml/common/types/modules.ts +++ b/x-pack/plugins/ml/common/types/modules.ts @@ -30,6 +30,7 @@ export interface KibanaObject { title: string; config: KibanaObjectConfig; exists?: boolean; + error?: any; } export interface KibanaObjects { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/components/kibana_objects.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/components/kibana_objects.tsx index 4954b44bf8842..f8ca7926ad7d6 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/components/kibana_objects.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/components/kibana_objects.tsx @@ -46,7 +46,7 @@ export const KibanaObjects: FC = memo(