From c64cba91f5fa3938b5e0317bcbc3a17ddc603f63 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 9 Jan 2019 14:48:25 -0700 Subject: [PATCH 01/13] task runner and usage collector for visualizations by type --- x-pack/index.js | 2 + x-pack/plugins/core_telemetry/constants.ts | 9 ++ x-pack/plugins/core_telemetry/index.d.ts | 66 ++++++++ x-pack/plugins/core_telemetry/index.js | 21 +++ .../server/lib/collectors/index.ts | 12 ++ .../get_usage_collector.test.ts | 58 +++++++ .../visualizations/get_usage_collector.ts | 34 +++++ .../register_usage_collector.ts | 14 ++ .../core_telemetry/server/lib/tasks/index.ts | 39 +++++ .../tasks/visualizations/task_runner.test.ts | 144 ++++++++++++++++++ .../lib/tasks/visualizations/task_runner.ts | 115 ++++++++++++++ .../core_telemetry/test_utils/index.ts | 57 +++++++ 12 files changed, 571 insertions(+) create mode 100644 x-pack/plugins/core_telemetry/constants.ts create mode 100644 x-pack/plugins/core_telemetry/index.d.ts create mode 100644 x-pack/plugins/core_telemetry/index.js create mode 100644 x-pack/plugins/core_telemetry/server/lib/collectors/index.ts create mode 100644 x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts create mode 100644 x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts create mode 100644 x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts create mode 100644 x-pack/plugins/core_telemetry/server/lib/tasks/index.ts create mode 100644 x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts create mode 100644 x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts create mode 100644 x-pack/plugins/core_telemetry/test_utils/index.ts diff --git a/x-pack/index.js b/x-pack/index.js index a395129d0880d..772a4ee0e5e9a 100644 --- a/x-pack/index.js +++ b/x-pack/index.js @@ -5,6 +5,7 @@ */ import { xpackMain } from './plugins/xpack_main'; +import { coreTelemetry } from './plugins/core_telemetry'; import { graph } from './plugins/graph'; import { monitoring } from './plugins/monitoring'; import { reporting } from './plugins/reporting'; @@ -39,6 +40,7 @@ import { uptime } from './plugins/uptime'; module.exports = function (kibana) { return [ xpackMain(kibana), + coreTelemetry(kibana), graph(kibana), monitoring(kibana), reporting(kibana), diff --git a/x-pack/plugins/core_telemetry/constants.ts b/x-pack/plugins/core_telemetry/constants.ts new file mode 100644 index 0000000000000..27f454a77be29 --- /dev/null +++ b/x-pack/plugins/core_telemetry/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const PLUGIN_ID = 'core_telemetry'; +export const VIS_TELEMETRY_TASK = 'vis_telemetry'; +export const VIS_USAGE_TYPE = 'visualization_types'; diff --git a/x-pack/plugins/core_telemetry/index.d.ts b/x-pack/plugins/core_telemetry/index.d.ts new file mode 100644 index 0000000000000..17131c5902dd4 --- /dev/null +++ b/x-pack/plugins/core_telemetry/index.d.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface IVisState { + type: string; +} + +export interface IVisualization { + visState: string; +} + +export interface ISavedObjectDoc { + _id: string; + _source: { + visualization: IVisualization; + type: string; + }; +} + +export interface IESQueryResponse { + hits: { + hits: ISavedObjectDoc[]; + }; +} + +export interface ITaskInstance { + state: { + runs: number; + stats: any; + }; + error?: any; +} + +export interface IHapiServer { + taskManager: { + registerTaskDefinitions: (opts: any) => void; + schedule: (opts: any) => Promise; + fetch: ( + opts: any + ) => Promise<{ + docs: ITaskInstance[]; + }>; + }; + plugins: { + xpack_main: any; + elasticsearch: { + getCluster: ( + cluster: string + ) => { + callWithInternalUser: () => Promise; + }; + }; + }; + usage: { + collectorSet: { + register: (collector: any) => void; + makeUsageCollector: (collectorOpts: any) => void; + }; + }; + config: () => { + get: (prop: string) => any; + }; +} diff --git a/x-pack/plugins/core_telemetry/index.js b/x-pack/plugins/core_telemetry/index.js new file mode 100644 index 0000000000000..3fa63871aeec5 --- /dev/null +++ b/x-pack/plugins/core_telemetry/index.js @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { registerCollectors } from './server/lib/collectors'; +import { registerTasks, scheduleTasks } from './server/lib/tasks'; + +export const coreTelemetry = (kibana) => { + return new kibana.Plugin({ + id: 'core_telemetry', + require: ['elasticsearch', 'xpack_main', 'task_manager'], + + init(server) { + registerCollectors(server); + registerTasks(server); + scheduleTasks(server); + } + }); +}; diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts new file mode 100644 index 0000000000000..a3e91357caeb6 --- /dev/null +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IHapiServer } from '../../../'; +import { getUsageCollector } from './visualizations/get_usage_collector'; + +export function registerCollectors(server: IHapiServer) { + getUsageCollector(server); +} diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts new file mode 100644 index 0000000000000..cca297b7dbbba --- /dev/null +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import sinon from 'sinon'; +import { IHapiServer } from '../../../../'; +import { + getMockCallWithInternal, + getMockKbnServer, + getMockTaskFetch, +} from '../../../../test_utils'; +import { getUsageCollector } from './get_usage_collector'; + +describe('getVisualizationsCollector#fetch', () => { + let mockKbnServer: IHapiServer; + + beforeEach(() => { + mockKbnServer = getMockKbnServer(getMockCallWithInternal(), getMockTaskFetch()); + }); + + test('can return empty stats', async () => { + const { type, fetch } = getUsageCollector(mockKbnServer); + expect(type).toBe('visualization_types'); + const fetchResult = await fetch(); + expect(fetchResult).toEqual({}); + }); + + test('provides known stats', async () => { + const mockTaskFetch = getMockTaskFetch([ + { + state: { + runs: 1, + stats: { comic_books: { total: 16, max: 12, min: 2, avg: 6 } }, + }, + }, + ]); + mockKbnServer = getMockKbnServer(getMockCallWithInternal(), mockTaskFetch); + + const { type, fetch } = getUsageCollector(mockKbnServer); + expect(type).toBe('visualization_types'); + const fetchResult = await fetch(); + expect(fetchResult).toEqual({ comic_books: { avg: 6, max: 12, min: 2, total: 16 } }); + }); + + describe('Error handling', () => { + // In real life, the CollectorSet calls fetch and handles errors + test('defers the errors', async () => { + const mockTaskFetch = sinon.stub(); + mockTaskFetch.rejects(new Error('BOOM')); + mockKbnServer = getMockKbnServer(getMockCallWithInternal(), mockTaskFetch); + + const { fetch } = getUsageCollector(mockKbnServer); + await expect(fetch()).rejects.toMatchObject(new Error('BOOM')); + }); + }); +}); diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts new file mode 100644 index 0000000000000..0af6893209a3f --- /dev/null +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; +import { IHapiServer } from '../../../../'; +import { PLUGIN_ID, VIS_TELEMETRY_TASK, VIS_USAGE_TYPE } from '../../../../constants'; + +export function getUsageCollector(server: IHapiServer) { + const { taskManager } = server; + return { + type: VIS_USAGE_TYPE, + fetch: async () => { + let docs; + try { + ({ docs } = await taskManager.fetch({ + bool: { filter: { term: { _id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}` } } }, + })); + } catch (err) { + if (err.constructor === Error && err.toString().indexOf('Error: NotInitialized') === 0) { + // it's fine + docs = {}; + } else { + throw err; + } + } + + // get the accumulated state from the recurring task + return get(docs, '[0].state.stats'); + }, + }; +} diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts new file mode 100644 index 0000000000000..7a5cf8056f829 --- /dev/null +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IHapiServer } from '../../../../'; +import { getUsageCollector } from './get_usage_collector'; + +export function getVisualizationsCollector(server: IHapiServer): void { + const { usage } = server; + const collector = usage.collectorSet.makeUsageCollector(getUsageCollector(server)); + server.usage.collectorSet.register(collector); +} diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts b/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts new file mode 100644 index 0000000000000..af2ed462e50d6 --- /dev/null +++ b/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IHapiServer } from '../../../'; +import { PLUGIN_ID, VIS_TELEMETRY_TASK } from '../../../constants'; +import { visualizationsTaskRunner } from './visualizations/task_runner'; + +export function registerTasks(server: IHapiServer) { + const { taskManager } = server; + + taskManager.registerTaskDefinitions({ + [VIS_TELEMETRY_TASK]: { + title: 'X-Pack telemetry calculator for Visualizations', + type: VIS_TELEMETRY_TASK, + numWorkers: 10, // use the max so no other tasks run concurrently + createTaskRunner({ taskInstance, kbnServer }: { kbnServer: any; taskInstance: any }) { + return { + run: visualizationsTaskRunner(taskInstance, kbnServer), + }; + }, + }, + }); +} + +export function scheduleTasks(server: IHapiServer) { + const { taskManager } = server; + const { kbnServer } = server.plugins.xpack_main.status.plugin; + + kbnServer.afterPluginsInit(() => { + taskManager.schedule({ + id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}`, + taskType: VIS_TELEMETRY_TASK, + state: { stats: {}, runs: 0 }, + }); + }); +} diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts new file mode 100644 index 0000000000000..165b18e817055 --- /dev/null +++ b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import moment from 'moment'; +import { IHapiServer, ITaskInstance } from '../../../../'; +import { + getMockCallWithInternal, + getMockKbnServer, + getMockTaskInstance, +} from '../../../../test_utils'; +import { visualizationsTaskRunner } from './task_runner'; + +describe('visualizationsTaskRunner', () => { + let mockTaskInstance: ITaskInstance; + let mockKbnServer: IHapiServer; + beforeEach(() => { + mockTaskInstance = getMockTaskInstance(); + mockKbnServer = getMockKbnServer(); + }); + + describe('Error handling', () => { + test('catches its own errors', async () => { + const mockCallWithInternal = () => Promise.reject(new Error('Things did not go well!')); + mockKbnServer = getMockKbnServer(mockCallWithInternal); + + const runner = visualizationsTaskRunner(mockTaskInstance, { server: mockKbnServer }); + const result = await runner(); + expect(result).toMatchObject({ + error: 'Things did not go well!', + state: { + runs: 1, + stats: undefined, + }, + }); + }); + }); + + test('Summarizes visualization response data', async () => { + const getNextMidnight = () => + moment() + .add(1, 'days') + .startOf('day') + .toISOString(); + + const runner = visualizationsTaskRunner(mockTaskInstance, { server: mockKbnServer }); + const result = await runner(); + + expect(result).toMatchObject({ + error: undefined, + runAt: getNextMidnight(), + state: { + runs: 1, + stats: { + shell_beads: { + spaces_avg: 1, + spaces_max: 1, + spaces_min: 1, + total: 1, + }, + }, + }, + }); + }); + + test('Summarizes visualization response data per Space', async () => { + const mockCallWithInternal = getMockCallWithInternal([ + // default space + { + _id: 'visualization:coolviz-123', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "cave_painting"}' }, + }, + }, + { + _id: 'visualization:coolviz-456', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "printing_press"}' }, + }, + }, + { + _id: 'meat:visualization:coolviz-789', + _source: { type: 'visualization', visualization: { visState: '{"type": "floppy_disk"}' } }, + }, + // meat space + { + _id: 'meat:visualization:coolviz-789', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "cave_painting"}' }, + }, + }, + { + _id: 'meat:visualization:coolviz-789', + _source: { type: 'visualization', visualization: { visState: '{"type": "cuneiform"}' } }, + }, + { + _id: 'meat:visualization:coolviz-789', + _source: { type: 'visualization', visualization: { visState: '{"type": "cuneiform"}' } }, + }, + { + _id: 'meat:visualization:coolviz-789', + _source: { type: 'visualization', visualization: { visState: '{"type": "floppy_disk"}' } }, + }, + // cyber space + { + _id: 'cyber:visualization:coolviz-789', + _source: { type: 'visualization', visualization: { visState: '{"type": "floppy_disk"}' } }, + }, + { + _id: 'cyber:visualization:coolviz-789', + _source: { type: 'visualization', visualization: { visState: '{"type": "floppy_disk"}' } }, + }, + { + _id: 'cyber:visualization:coolviz-123', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "cave_painting"}' }, + }, + }, + ]); + mockKbnServer = getMockKbnServer(mockCallWithInternal); + + const runner = visualizationsTaskRunner(mockTaskInstance, { server: mockKbnServer }); + const result = await runner(); + + expect(result).toMatchObject({ + error: undefined, + state: { + runs: 1, + stats: { + cave_painting: { total: 3, spaces_min: 1, spaces_max: 1, spaces_avg: 1 }, + printing_press: { total: 1, spaces_min: 1, spaces_max: 1, spaces_avg: 1 }, + cuneiform: { total: 2, spaces_min: 2, spaces_max: 2, spaces_avg: 2 }, + floppy_disk: { total: 4, spaces_min: 2, spaces_max: 2, spaces_avg: 2 }, + }, + }, + }); + }); +}); diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts new file mode 100644 index 0000000000000..34b52cf49a763 --- /dev/null +++ b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import _, { countBy, groupBy } from 'lodash'; +import moment from 'moment'; +import { + IESQueryResponse, + IHapiServer, + ISavedObjectDoc, + ITaskInstance, + IVisState, + IVisualization, +} from '../../../../'; + +interface IVisSummary { + type: string; + space: string; +} + +/* + * Parse the response data into telemetry payload + */ +async function getStats(callCluster: (method: string, params: any) => Promise, index: string) { + const searchParams = { + size: 10000, // elasticsearch index.max_result_window default value + index, + ignoreUnavailable: true, + filterPath: ['hits.hits._id', 'hits.hits._source.visualization'], + body: { + query: { + bool: { filter: { term: { type: 'visualization' } } }, + }, + }, + }; + const esResponse: IESQueryResponse = await callCluster('search', searchParams); + const size = _.get(esResponse, 'hits.hits.length'); + if (size < 1) { + return; + } + + // `map` to get the raw types + const visSummaries: IVisSummary[] = esResponse.hits.hits.map((hit: ISavedObjectDoc) => { + const spacePhrases: string[] = hit._id.split(':'); + const space = spacePhrases.length === 3 ? spacePhrases[0] : 'default'; // if in a custom space, the format of a saved object ID is space:type:id + const visualization: IVisualization = _.get(hit, '_source.visualization', { visState: '{}' }); + const visState: IVisState = JSON.parse(visualization.visState); + + return { + type: visState.type || '_na_', + space, + }; + }); + + // organize stats per type + const visTypes = groupBy(visSummaries, 'type'); + + // get the final result + return Object.keys(visTypes).reduce((accum, curr) => { + const total = visTypes[curr].length; + const spacesBreakdown = countBy(visTypes[curr], 'space'); + const spaceCounts: number[] = _.values(spacesBreakdown); + + return { + ...accum, + [curr]: { + total, + spaces_min: _.min(spaceCounts), + spaces_max: _.max(spaceCounts), + spaces_avg: total / spaceCounts.length, + }, + }; + }, {}); +} + +export function visualizationsTaskRunner( + taskInstance: ITaskInstance, + kbnServer: { server: IHapiServer } +) { + const { server } = kbnServer; + const { callWithInternalUser: callCluster } = server.plugins.elasticsearch.getCluster('data'); + const config = server.config(); + const index = config.get('kibana.index').toString(); // cast to string for TypeScript + + return async () => { + let stats; + let error; + + try { + stats = await getStats(callCluster, index); + } catch (err) { + if (err.constructor === Error) { + error = err.message; + } else { + error = err; + } + } + + const nextMidnight = moment() + .add(1, 'days') + .startOf('day') + .toISOString(); + + return { + runAt: nextMidnight, + state: { + runs: taskInstance.state.runs + 1, + stats, + }, + error, + }; + }; +} diff --git a/x-pack/plugins/core_telemetry/test_utils/index.ts b/x-pack/plugins/core_telemetry/test_utils/index.ts new file mode 100644 index 0000000000000..8919d9207ad8a --- /dev/null +++ b/x-pack/plugins/core_telemetry/test_utils/index.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IESQueryResponse, IHapiServer, ISavedObjectDoc, ITaskInstance } from '../'; + +export const getMockTaskInstance = (): ITaskInstance => ({ state: { runs: 0, stats: {} } }); + +const defaultMockSavedObjects = [ + { + _id: 'visualization:coolviz-123', + _source: { + type: 'visualization', + visualization: { visState: '{"type": "shell_beads"}' }, + }, + }, +]; + +const defaultMockTaskDocs = [getMockTaskInstance()]; + +export const getMockCallWithInternal = (hits: ISavedObjectDoc[] = defaultMockSavedObjects) => { + return (): Promise => { + return Promise.resolve({ hits: { hits } }); + }; +}; + +export const getMockTaskFetch = (docs: ITaskInstance[] = defaultMockTaskDocs) => { + return () => Promise.resolve({ docs }); +}; + +export const getMockKbnServer = ( + mockCallWithInternal = getMockCallWithInternal(), + mockTaskFetch = getMockTaskFetch() +): IHapiServer => ({ + taskManager: { + registerTaskDefinitions: (opts: any) => undefined, + schedule: (opts: any) => Promise.resolve(), + fetch: mockTaskFetch, + }, + plugins: { + elasticsearch: { + getCluster: (cluster: string) => ({ + callWithInternalUser: mockCallWithInternal, + }), + }, + xpack_main: {}, + }, + usage: { + collectorSet: { + makeUsageCollector: () => '', + register: () => undefined, + }, + }, + config: () => ({ get: () => '' }), +}); From e5c000454b15ab7ec6a5a18b8cc49f0633247d3d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Thu, 17 Jan 2019 22:01:08 -0700 Subject: [PATCH 02/13] type is always just "visualization" --- x-pack/plugins/core_telemetry/index.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/core_telemetry/index.d.ts b/x-pack/plugins/core_telemetry/index.d.ts index 17131c5902dd4..b3942fa54b016 100644 --- a/x-pack/plugins/core_telemetry/index.d.ts +++ b/x-pack/plugins/core_telemetry/index.d.ts @@ -16,7 +16,7 @@ export interface ISavedObjectDoc { _id: string; _source: { visualization: IVisualization; - type: string; + type: 'visualization'; }; } From 7e8adde3607fb27c3d340d7558a0c9cda8b6ad6c Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 15:41:34 -0700 Subject: [PATCH 03/13] drop the I- prefix for interfaces --- x-pack/plugins/core_telemetry/constants.ts | 6 ++--- x-pack/plugins/core_telemetry/index.d.ts | 20 +++++++------- .../server/lib/collectors/index.ts | 4 +-- .../get_usage_collector.test.ts | 4 +-- .../visualizations/get_usage_collector.ts | 11 +++++--- .../register_usage_collector.ts | 4 +-- .../core_telemetry/server/lib/tasks/index.ts | 6 ++--- .../tasks/visualizations/task_runner.test.ts | 6 ++--- .../lib/tasks/visualizations/task_runner.ts | 26 +++++++++---------- .../core_telemetry/test_utils/index.ts | 12 ++++----- 10 files changed, 51 insertions(+), 48 deletions(-) diff --git a/x-pack/plugins/core_telemetry/constants.ts b/x-pack/plugins/core_telemetry/constants.ts index 27f454a77be29..bac5c25364cd9 100644 --- a/x-pack/plugins/core_telemetry/constants.ts +++ b/x-pack/plugins/core_telemetry/constants.ts @@ -4,6 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -export const PLUGIN_ID = 'core_telemetry'; -export const VIS_TELEMETRY_TASK = 'vis_telemetry'; -export const VIS_USAGE_TYPE = 'visualization_types'; +export const PLUGIN_ID = 'core_telemetry'; // prefix used for registering properties with services from this plugin +export const VIS_TELEMETRY_TASK = 'vis_telemetry'; // suffix for the _id of our task instance, which must be `get`-able +export const VIS_USAGE_TYPE = 'visualization_types'; // suffix for the properties of data registered with the usage service diff --git a/x-pack/plugins/core_telemetry/index.d.ts b/x-pack/plugins/core_telemetry/index.d.ts index b3942fa54b016..925cab9c44682 100644 --- a/x-pack/plugins/core_telemetry/index.d.ts +++ b/x-pack/plugins/core_telemetry/index.d.ts @@ -4,29 +4,29 @@ * you may not use this file except in compliance with the Elastic License. */ -export interface IVisState { +export interface VisState { type: string; } -export interface IVisualization { +export interface Visualization { visState: string; } -export interface ISavedObjectDoc { +export interface SavedObjectDoc { _id: string; _source: { - visualization: IVisualization; + visualization: Visualization; type: 'visualization'; }; } -export interface IESQueryResponse { +export interface ESQueryResponse { hits: { - hits: ISavedObjectDoc[]; + hits: SavedObjectDoc[]; }; } -export interface ITaskInstance { +export interface TaskInstance { state: { runs: number; stats: any; @@ -34,14 +34,14 @@ export interface ITaskInstance { error?: any; } -export interface IHapiServer { +export interface HapiServer { taskManager: { registerTaskDefinitions: (opts: any) => void; schedule: (opts: any) => Promise; fetch: ( opts: any ) => Promise<{ - docs: ITaskInstance[]; + docs: TaskInstance[]; }>; }; plugins: { @@ -50,7 +50,7 @@ export interface IHapiServer { getCluster: ( cluster: string ) => { - callWithInternalUser: () => Promise; + callWithInternalUser: () => Promise; }; }; }; diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts index a3e91357caeb6..d35ccaa3b6115 100644 --- a/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IHapiServer } from '../../../'; +import { HapiServer } from '../../../'; import { getUsageCollector } from './visualizations/get_usage_collector'; -export function registerCollectors(server: IHapiServer) { +export function registerCollectors(server: HapiServer) { getUsageCollector(server); } diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts index cca297b7dbbba..d26655ee0f4d2 100644 --- a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts @@ -5,7 +5,7 @@ */ import sinon from 'sinon'; -import { IHapiServer } from '../../../../'; +import { HapiServer } from '../../../../'; import { getMockCallWithInternal, getMockKbnServer, @@ -14,7 +14,7 @@ import { import { getUsageCollector } from './get_usage_collector'; describe('getVisualizationsCollector#fetch', () => { - let mockKbnServer: IHapiServer; + let mockKbnServer: HapiServer; beforeEach(() => { mockKbnServer = getMockKbnServer(getMockCallWithInternal(), getMockTaskFetch()); diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts index 0af6893209a3f..833c2388f14d8 100644 --- a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts @@ -5,10 +5,10 @@ */ import { get } from 'lodash'; -import { IHapiServer } from '../../../../'; +import { HapiServer } from '../../../../'; import { PLUGIN_ID, VIS_TELEMETRY_TASK, VIS_USAGE_TYPE } from '../../../../constants'; -export function getUsageCollector(server: IHapiServer) { +export function getUsageCollector(server: HapiServer) { const { taskManager } = server; return { type: VIS_USAGE_TYPE, @@ -19,8 +19,11 @@ export function getUsageCollector(server: IHapiServer) { bool: { filter: { term: { _id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}` } } }, })); } catch (err) { - if (err.constructor === Error && err.toString().indexOf('Error: NotInitialized') === 0) { - // it's fine + const errMessage = err && err.message ? err.message : err.toString(); + if (errMessage.indexOf('NotInitialized') >= 0) { + // it's possible for the usage service to try to fetch from this collector before the task manager has been + // initialized. if we catch this particular case, it's fine to ignore it: next time around it will be + // initialized (or throw a different type of error) docs = {}; } else { throw err; diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts index 7a5cf8056f829..903c3e6ed6ff7 100644 --- a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IHapiServer } from '../../../../'; +import { HapiServer } from '../../../../'; import { getUsageCollector } from './get_usage_collector'; -export function getVisualizationsCollector(server: IHapiServer): void { +export function getVisualizationsCollector(server: HapiServer): void { const { usage } = server; const collector = usage.collectorSet.makeUsageCollector(getUsageCollector(server)); server.usage.collectorSet.register(collector); diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts b/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts index af2ed462e50d6..f8970db67973d 100644 --- a/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts +++ b/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts @@ -4,11 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IHapiServer } from '../../../'; +import { HapiServer } from '../../../'; import { PLUGIN_ID, VIS_TELEMETRY_TASK } from '../../../constants'; import { visualizationsTaskRunner } from './visualizations/task_runner'; -export function registerTasks(server: IHapiServer) { +export function registerTasks(server: HapiServer) { const { taskManager } = server; taskManager.registerTaskDefinitions({ @@ -25,7 +25,7 @@ export function registerTasks(server: IHapiServer) { }); } -export function scheduleTasks(server: IHapiServer) { +export function scheduleTasks(server: HapiServer) { const { taskManager } = server; const { kbnServer } = server.plugins.xpack_main.status.plugin; diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts index 165b18e817055..879846aeed1ae 100644 --- a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts +++ b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts @@ -5,7 +5,7 @@ */ import moment from 'moment'; -import { IHapiServer, ITaskInstance } from '../../../../'; +import { HapiServer, TaskInstance } from '../../../../'; import { getMockCallWithInternal, getMockKbnServer, @@ -14,8 +14,8 @@ import { import { visualizationsTaskRunner } from './task_runner'; describe('visualizationsTaskRunner', () => { - let mockTaskInstance: ITaskInstance; - let mockKbnServer: IHapiServer; + let mockTaskInstance: TaskInstance; + let mockKbnServer: HapiServer; beforeEach(() => { mockTaskInstance = getMockTaskInstance(); mockKbnServer = getMockKbnServer(); diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts index 34b52cf49a763..30f0dbfc703ae 100644 --- a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts +++ b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts @@ -7,15 +7,15 @@ import _, { countBy, groupBy } from 'lodash'; import moment from 'moment'; import { - IESQueryResponse, - IHapiServer, - ISavedObjectDoc, - ITaskInstance, - IVisState, - IVisualization, + ESQueryResponse, + HapiServer, + SavedObjectDoc, + TaskInstance, + VisState, + Visualization, } from '../../../../'; -interface IVisSummary { +interface VisSummary { type: string; space: string; } @@ -35,18 +35,18 @@ async function getStats(callCluster: (method: string, params: any) => Promise { + const visSummaries: VisSummary[] = esResponse.hits.hits.map((hit: SavedObjectDoc) => { const spacePhrases: string[] = hit._id.split(':'); const space = spacePhrases.length === 3 ? spacePhrases[0] : 'default'; // if in a custom space, the format of a saved object ID is space:type:id - const visualization: IVisualization = _.get(hit, '_source.visualization', { visState: '{}' }); - const visState: IVisState = JSON.parse(visualization.visState); + const visualization: Visualization = _.get(hit, '_source.visualization', { visState: '{}' }); + const visState: VisState = JSON.parse(visualization.visState); return { type: visState.type || '_na_', @@ -76,8 +76,8 @@ async function getStats(callCluster: (method: string, params: any) => Promise ({ state: { runs: 0, stats: {} } }); +export const getMockTaskInstance = (): TaskInstance => ({ state: { runs: 0, stats: {} } }); const defaultMockSavedObjects = [ { @@ -20,20 +20,20 @@ const defaultMockSavedObjects = [ const defaultMockTaskDocs = [getMockTaskInstance()]; -export const getMockCallWithInternal = (hits: ISavedObjectDoc[] = defaultMockSavedObjects) => { - return (): Promise => { +export const getMockCallWithInternal = (hits: SavedObjectDoc[] = defaultMockSavedObjects) => { + return (): Promise => { return Promise.resolve({ hits: { hits } }); }; }; -export const getMockTaskFetch = (docs: ITaskInstance[] = defaultMockTaskDocs) => { +export const getMockTaskFetch = (docs: TaskInstance[] = defaultMockTaskDocs) => { return () => Promise.resolve({ docs }); }; export const getMockKbnServer = ( mockCallWithInternal = getMockCallWithInternal(), mockTaskFetch = getMockTaskFetch() -): IHapiServer => ({ +): HapiServer => ({ taskManager: { registerTaskDefinitions: (opts: any) => undefined, schedule: (opts: any) => Promise.resolve(), From c99957788a8265da8e5127eaf2d8c267c041be52 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 17:16:11 -0700 Subject: [PATCH 04/13] bug fixes --- x-pack/plugins/core_telemetry/server/lib/collectors/index.ts | 4 ++-- .../lib/collectors/visualizations/get_usage_collector.ts | 2 +- .../lib/collectors/visualizations/register_usage_collector.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts index d35ccaa3b6115..8b825b13178f2 100644 --- a/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts @@ -5,8 +5,8 @@ */ import { HapiServer } from '../../../'; -import { getUsageCollector } from './visualizations/get_usage_collector'; +import { registerVisualizationsCollector } from './visualizations/register_usage_collector'; export function registerCollectors(server: HapiServer) { - getUsageCollector(server); + registerVisualizationsCollector(server); } diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts index 833c2388f14d8..33a53c3fb1db6 100644 --- a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts @@ -16,7 +16,7 @@ export function getUsageCollector(server: HapiServer) { let docs; try { ({ docs } = await taskManager.fetch({ - bool: { filter: { term: { _id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}` } } }, + query: { bool: { filter: { term: { _id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}` } } } }, })); } catch (err) { const errMessage = err && err.message ? err.message : err.toString(); diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts index 903c3e6ed6ff7..0a26b0e47494b 100644 --- a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts @@ -7,7 +7,7 @@ import { HapiServer } from '../../../../'; import { getUsageCollector } from './get_usage_collector'; -export function getVisualizationsCollector(server: HapiServer): void { +export function registerVisualizationsCollector(server: HapiServer): void { const { usage } = server; const collector = usage.collectorSet.makeUsageCollector(getUsageCollector(server)); server.usage.collectorSet.register(collector); From 6e14bd83f1535bb9235cd634d81f34a189353fe5 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 17:18:36 -0700 Subject: [PATCH 05/13] ts fix --- x-pack/plugins/core_telemetry/index.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/core_telemetry/index.d.ts b/x-pack/plugins/core_telemetry/index.d.ts index 925cab9c44682..0a345844e4b40 100644 --- a/x-pack/plugins/core_telemetry/index.d.ts +++ b/x-pack/plugins/core_telemetry/index.d.ts @@ -16,7 +16,7 @@ export interface SavedObjectDoc { _id: string; _source: { visualization: Visualization; - type: 'visualization'; + type: string; }; } From ddb96858b6719a030160ef50b8baf326e7609f91 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 17:21:30 -0700 Subject: [PATCH 06/13] comment perfection --- .../lib/collectors/visualizations/get_usage_collector.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts index 33a53c3fb1db6..734f3843a455a 100644 --- a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts @@ -20,10 +20,12 @@ export function getUsageCollector(server: HapiServer) { })); } catch (err) { const errMessage = err && err.message ? err.message : err.toString(); + /* + * The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the task manager + * has to wait for all plugins to initialize first. + * It's fine to ignore it as next time around it will be initialized (or it will throw a different type of error) + */ if (errMessage.indexOf('NotInitialized') >= 0) { - // it's possible for the usage service to try to fetch from this collector before the task manager has been - // initialized. if we catch this particular case, it's fine to ignore it: next time around it will be - // initialized (or throw a different type of error) docs = {}; } else { throw err; From d648e7926cd9b8b3fa977b26bc47d786b343f7fe Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 17:22:10 -0700 Subject: [PATCH 07/13] just usage. --- .../lib/collectors/visualizations/register_usage_collector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts index 0a26b0e47494b..555c7ac27b49d 100644 --- a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts +++ b/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts @@ -10,5 +10,5 @@ import { getUsageCollector } from './get_usage_collector'; export function registerVisualizationsCollector(server: HapiServer): void { const { usage } = server; const collector = usage.collectorSet.makeUsageCollector(getUsageCollector(server)); - server.usage.collectorSet.register(collector); + usage.collectorSet.register(collector); } From dad1216e596031fd7947e8d270498f1e92160238 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 17:26:18 -0700 Subject: [PATCH 08/13] const for task numworkers --- x-pack/plugins/core_telemetry/constants.ts | 2 ++ x-pack/plugins/core_telemetry/server/lib/tasks/index.ts | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/core_telemetry/constants.ts b/x-pack/plugins/core_telemetry/constants.ts index bac5c25364cd9..11bb5b0018757 100644 --- a/x-pack/plugins/core_telemetry/constants.ts +++ b/x-pack/plugins/core_telemetry/constants.ts @@ -7,3 +7,5 @@ export const PLUGIN_ID = 'core_telemetry'; // prefix used for registering properties with services from this plugin export const VIS_TELEMETRY_TASK = 'vis_telemetry'; // suffix for the _id of our task instance, which must be `get`-able export const VIS_USAGE_TYPE = 'visualization_types'; // suffix for the properties of data registered with the usage service + +export const VIS_TELEMETRY_TASK_NUM_WORKERS = 10; // by default it's 100% their workers. Users can scale up and set task manager's numWorkers higher for other tasks to be able to run concurrently in a single Kibana instance with this one diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts b/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts index f8970db67973d..fac51256c9b2a 100644 --- a/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts +++ b/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts @@ -5,7 +5,7 @@ */ import { HapiServer } from '../../../'; -import { PLUGIN_ID, VIS_TELEMETRY_TASK } from '../../../constants'; +import { PLUGIN_ID, VIS_TELEMETRY_TASK, VIS_TELEMETRY_TASK_NUM_WORKERS } from '../../../constants'; import { visualizationsTaskRunner } from './visualizations/task_runner'; export function registerTasks(server: HapiServer) { @@ -15,7 +15,7 @@ export function registerTasks(server: HapiServer) { [VIS_TELEMETRY_TASK]: { title: 'X-Pack telemetry calculator for Visualizations', type: VIS_TELEMETRY_TASK, - numWorkers: 10, // use the max so no other tasks run concurrently + numWorkers: VIS_TELEMETRY_TASK_NUM_WORKERS, // by default it's 100% their workers createTaskRunner({ taskInstance, kbnServer }: { kbnServer: any; taskInstance: any }) { return { run: visualizationsTaskRunner(taskInstance, kbnServer), From b26dda48763bceda3102ad2c829922b776f3af8c Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 17:33:19 -0700 Subject: [PATCH 09/13] use mapValues --- .../lib/tasks/visualizations/task_runner.ts | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts index 30f0dbfc703ae..fd8c11a62a43b 100644 --- a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts +++ b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import _, { countBy, groupBy } from 'lodash'; +import _, { countBy, groupBy, mapValues } from 'lodash'; import moment from 'moment'; import { ESQueryResponse, @@ -58,21 +58,18 @@ async function getStats(callCluster: (method: string, params: any) => Promise { - const total = visTypes[curr].length; - const spacesBreakdown = countBy(visTypes[curr], 'space'); + return mapValues(visTypes, curr => { + const total = curr.length; + const spacesBreakdown = countBy(curr, 'space'); const spaceCounts: number[] = _.values(spacesBreakdown); return { - ...accum, - [curr]: { - total, - spaces_min: _.min(spaceCounts), - spaces_max: _.max(spaceCounts), - spaces_avg: total / spaceCounts.length, - }, + total, + spaces_min: _.min(spaceCounts), + spaces_max: _.max(spaceCounts), + spaces_avg: total / spaceCounts.length, }; - }, {}); + }); } export function visualizationsTaskRunner( From 75eeb160978bbd9298cca33823354cb8ed9db168 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 17:41:25 -0700 Subject: [PATCH 10/13] get next midnight module --- .../server/lib/get_next_midnight.test.ts | 17 +++++++++++++++++ .../server/lib/get_next_midnight.ts | 12 ++++++++++++ .../lib/tasks/visualizations/task_runner.ts | 9 ++------- 3 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/core_telemetry/server/lib/get_next_midnight.test.ts create mode 100644 x-pack/plugins/core_telemetry/server/lib/get_next_midnight.ts diff --git a/x-pack/plugins/core_telemetry/server/lib/get_next_midnight.test.ts b/x-pack/plugins/core_telemetry/server/lib/get_next_midnight.test.ts new file mode 100644 index 0000000000000..2aa1fa6985722 --- /dev/null +++ b/x-pack/plugins/core_telemetry/server/lib/get_next_midnight.test.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import moment from 'moment'; +import { getNextMidnight } from './get_next_midnight'; + +describe('getNextMidnight', () => { + const nextMidnightMoment = moment() + .add(1, 'days') + .startOf('day') + .toISOString(); + + expect(getNextMidnight()).toEqual(nextMidnightMoment); +}); diff --git a/x-pack/plugins/core_telemetry/server/lib/get_next_midnight.ts b/x-pack/plugins/core_telemetry/server/lib/get_next_midnight.ts new file mode 100644 index 0000000000000..c286af1854b6a --- /dev/null +++ b/x-pack/plugins/core_telemetry/server/lib/get_next_midnight.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export function getNextMidnight() { + const nextMidnight = new Date(); + nextMidnight.setHours(0, 0, 0, 0); + nextMidnight.setDate(nextMidnight.getDate() + 1); + return nextMidnight.toISOString(); +} diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts index fd8c11a62a43b..0cd06275546b4 100644 --- a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts +++ b/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts @@ -5,7 +5,6 @@ */ import _, { countBy, groupBy, mapValues } from 'lodash'; -import moment from 'moment'; import { ESQueryResponse, HapiServer, @@ -14,6 +13,7 @@ import { VisState, Visualization, } from '../../../../'; +import { getNextMidnight } from '../../get_next_midnight'; interface VisSummary { type: string; @@ -95,13 +95,8 @@ export function visualizationsTaskRunner( } } - const nextMidnight = moment() - .add(1, 'days') - .startOf('day') - .toISOString(); - return { - runAt: nextMidnight, + runAt: getNextMidnight(), state: { runs: taskInstance.state.runs + 1, stats, From ed77b194b844b5a77d46d932ff2c576de607f325 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 17:45:42 -0700 Subject: [PATCH 11/13] move to oss_telemtry --- x-pack/index.js | 4 ++-- .../plugins/{core_telemetry => oss_telemetry}/constants.ts | 2 +- x-pack/plugins/{core_telemetry => oss_telemetry}/index.d.ts | 0 x-pack/plugins/{core_telemetry => oss_telemetry}/index.js | 5 +++-- .../server/lib/collectors/index.ts | 0 .../collectors/visualizations/get_usage_collector.test.ts | 0 .../lib/collectors/visualizations/get_usage_collector.ts | 0 .../collectors/visualizations/register_usage_collector.ts | 0 .../server/lib/get_next_midnight.test.ts | 0 .../server/lib/get_next_midnight.ts | 0 .../server/lib/tasks/index.ts | 0 .../server/lib/tasks/visualizations/task_runner.test.ts | 0 .../server/lib/tasks/visualizations/task_runner.ts | 0 .../{core_telemetry => oss_telemetry}/test_utils/index.ts | 0 14 files changed, 6 insertions(+), 5 deletions(-) rename x-pack/plugins/{core_telemetry => oss_telemetry}/constants.ts (86%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/index.d.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/index.js (85%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/server/lib/collectors/index.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/server/lib/collectors/visualizations/get_usage_collector.test.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/server/lib/collectors/visualizations/get_usage_collector.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/server/lib/collectors/visualizations/register_usage_collector.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/server/lib/get_next_midnight.test.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/server/lib/get_next_midnight.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/server/lib/tasks/index.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/server/lib/tasks/visualizations/task_runner.test.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/server/lib/tasks/visualizations/task_runner.ts (100%) rename x-pack/plugins/{core_telemetry => oss_telemetry}/test_utils/index.ts (100%) diff --git a/x-pack/index.js b/x-pack/index.js index 772a4ee0e5e9a..148f624a09220 100644 --- a/x-pack/index.js +++ b/x-pack/index.js @@ -5,7 +5,6 @@ */ import { xpackMain } from './plugins/xpack_main'; -import { coreTelemetry } from './plugins/core_telemetry'; import { graph } from './plugins/graph'; import { monitoring } from './plugins/monitoring'; import { reporting } from './plugins/reporting'; @@ -36,11 +35,11 @@ import { remoteClusters } from './plugins/remote_clusters'; import { crossClusterReplication } from './plugins/cross_cluster_replication'; import { upgradeAssistant } from './plugins/upgrade_assistant'; import { uptime } from './plugins/uptime'; +import { ossTelemetry } from './plugins/oss_telemetry'; module.exports = function (kibana) { return [ xpackMain(kibana), - coreTelemetry(kibana), graph(kibana), monitoring(kibana), reporting(kibana), @@ -71,5 +70,6 @@ module.exports = function (kibana) { crossClusterReplication(kibana), upgradeAssistant(kibana), uptime(kibana), + ossTelemetry(kibana), ]; }; diff --git a/x-pack/plugins/core_telemetry/constants.ts b/x-pack/plugins/oss_telemetry/constants.ts similarity index 86% rename from x-pack/plugins/core_telemetry/constants.ts rename to x-pack/plugins/oss_telemetry/constants.ts index 11bb5b0018757..4453f7d543784 100644 --- a/x-pack/plugins/core_telemetry/constants.ts +++ b/x-pack/plugins/oss_telemetry/constants.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -export const PLUGIN_ID = 'core_telemetry'; // prefix used for registering properties with services from this plugin +export const PLUGIN_ID = 'oss_telemetry'; // prefix used for registering properties with services from this plugin export const VIS_TELEMETRY_TASK = 'vis_telemetry'; // suffix for the _id of our task instance, which must be `get`-able export const VIS_USAGE_TYPE = 'visualization_types'; // suffix for the properties of data registered with the usage service diff --git a/x-pack/plugins/core_telemetry/index.d.ts b/x-pack/plugins/oss_telemetry/index.d.ts similarity index 100% rename from x-pack/plugins/core_telemetry/index.d.ts rename to x-pack/plugins/oss_telemetry/index.d.ts diff --git a/x-pack/plugins/core_telemetry/index.js b/x-pack/plugins/oss_telemetry/index.js similarity index 85% rename from x-pack/plugins/core_telemetry/index.js rename to x-pack/plugins/oss_telemetry/index.js index 3fa63871aeec5..aeb7f709421bf 100644 --- a/x-pack/plugins/core_telemetry/index.js +++ b/x-pack/plugins/oss_telemetry/index.js @@ -6,10 +6,11 @@ import { registerCollectors } from './server/lib/collectors'; import { registerTasks, scheduleTasks } from './server/lib/tasks'; +import { PLUGIN_ID } from './constants'; -export const coreTelemetry = (kibana) => { +export const ossTelemetry = (kibana) => { return new kibana.Plugin({ - id: 'core_telemetry', + id: PLUGIN_ID, require: ['elasticsearch', 'xpack_main', 'task_manager'], init(server) { diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/index.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/index.ts similarity index 100% rename from x-pack/plugins/core_telemetry/server/lib/collectors/index.ts rename to x-pack/plugins/oss_telemetry/server/lib/collectors/index.ts diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts similarity index 100% rename from x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts rename to x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts similarity index 100% rename from x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts rename to x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts diff --git a/x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts similarity index 100% rename from x-pack/plugins/core_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts rename to x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/register_usage_collector.ts diff --git a/x-pack/plugins/core_telemetry/server/lib/get_next_midnight.test.ts b/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts similarity index 100% rename from x-pack/plugins/core_telemetry/server/lib/get_next_midnight.test.ts rename to x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts diff --git a/x-pack/plugins/core_telemetry/server/lib/get_next_midnight.ts b/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.ts similarity index 100% rename from x-pack/plugins/core_telemetry/server/lib/get_next_midnight.ts rename to x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.ts diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/index.ts b/x-pack/plugins/oss_telemetry/server/lib/tasks/index.ts similarity index 100% rename from x-pack/plugins/core_telemetry/server/lib/tasks/index.ts rename to x-pack/plugins/oss_telemetry/server/lib/tasks/index.ts diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts b/x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts similarity index 100% rename from x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.test.ts rename to x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts diff --git a/x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts b/x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.ts similarity index 100% rename from x-pack/plugins/core_telemetry/server/lib/tasks/visualizations/task_runner.ts rename to x-pack/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.ts diff --git a/x-pack/plugins/core_telemetry/test_utils/index.ts b/x-pack/plugins/oss_telemetry/test_utils/index.ts similarity index 100% rename from x-pack/plugins/core_telemetry/test_utils/index.ts rename to x-pack/plugins/oss_telemetry/test_utils/index.ts From fef2cb99f58331100bfd988612e42a315895bd93 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sat, 26 Jan 2019 17:48:05 -0700 Subject: [PATCH 12/13] test fix --- .../server/lib/get_next_midnight.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts b/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts index 2aa1fa6985722..d8df006a55b7f 100644 --- a/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts +++ b/x-pack/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts @@ -8,10 +8,12 @@ import moment from 'moment'; import { getNextMidnight } from './get_next_midnight'; describe('getNextMidnight', () => { - const nextMidnightMoment = moment() - .add(1, 'days') - .startOf('day') - .toISOString(); + test('Returns the next time and date of midnight as an iso string', () => { + const nextMidnightMoment = moment() + .add(1, 'days') + .startOf('day') + .toISOString(); - expect(getNextMidnight()).toEqual(nextMidnightMoment); + expect(getNextMidnight()).toEqual(nextMidnightMoment); + }); }); From 37a0f2f3788a5f70636d87c8cd41e143e39f907c Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 29 Jan 2019 22:01:03 -0700 Subject: [PATCH 13/13] errMessage.includes(NotInitialized) --- .../visualizations/get_usage_collector.test.ts | 10 ++++++++++ .../collectors/visualizations/get_usage_collector.ts | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts index d26655ee0f4d2..d316562c826d6 100644 --- a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts +++ b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.test.ts @@ -45,6 +45,16 @@ describe('getVisualizationsCollector#fetch', () => { }); describe('Error handling', () => { + test('Silently handles Task Manager NotInitialized', async () => { + const mockTaskFetch = sinon.stub(); + mockTaskFetch.rejects( + new Error('NotInitialized taskManager is still waiting for plugins to load') + ); + mockKbnServer = getMockKbnServer(getMockCallWithInternal(), mockTaskFetch); + + const { fetch } = getUsageCollector(mockKbnServer); + await expect(fetch()).resolves.toBe(undefined); + }); // In real life, the CollectorSet calls fetch and handles errors test('defers the errors', async () => { const mockTaskFetch = sinon.stub(); diff --git a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts index 734f3843a455a..159ca7eec8063 100644 --- a/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts +++ b/x-pack/plugins/oss_telemetry/server/lib/collectors/visualizations/get_usage_collector.ts @@ -25,7 +25,7 @@ export function getUsageCollector(server: HapiServer) { * has to wait for all plugins to initialize first. * It's fine to ignore it as next time around it will be initialized (or it will throw a different type of error) */ - if (errMessage.indexOf('NotInitialized') >= 0) { + if (errMessage.includes('NotInitialized')) { docs = {}; } else { throw err;