From fb717d7892c91eeeb6c3f2f63b978a0ee65e306b Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Fri, 9 Apr 2021 14:21:41 +0300 Subject: [PATCH 01/11] finish implementation and tests --- .../server/collectors/index.ts | 4 + .../__fixtures__/ui_counter_saved_objects.ts | 41 +++ .../usage_counter_saved_objects.ts | 72 +++++ .../register_ui_counters_collector.test.ts | 192 ++++++++---- .../register_ui_counters_collector.ts | 150 ++++++++-- .../ui_counters/rollups/register_rollups.ts | 13 +- .../ui_counters/rollups/rollups.test.ts | 34 ++- .../collectors/ui_counters/rollups/rollups.ts | 16 + .../usage_counter_saved_objects.ts | 82 ++++++ .../server/collectors/usage_counters/index.ts | 10 + .../register_ui_counters_collector.test.ts | 55 ++++ .../register_usage_counters_collector.ts | 119 ++++++++ .../usage_counters/rollups/constants.ts | 22 ++ .../usage_counters/rollups/index.ts | 9 + .../rollups/register_rollups.ts | 21 ++ .../usage_counters/rollups/rollups.test.ts | 166 +++++++++++ .../usage_counters/rollups/rollups.ts | 73 +++++ .../kibana_usage_collection/server/plugin.ts | 7 + src/plugins/telemetry/schema/oss_plugins.json | 47 +++ .../usage_collection/common/ui_counters.ts | 23 ++ .../server/collector/collector_set.ts | 32 +- .../server/collector/index.ts | 2 +- src/plugins/usage_collection/server/config.ts | 5 + src/plugins/usage_collection/server/index.ts | 12 + src/plugins/usage_collection/server/plugin.ts | 85 +++++- .../server/report/store_report.test.ts | 81 ++++-- .../server/report/store_report.ts | 18 +- .../usage_collection/server/routes/index.ts | 6 +- .../server/routes/ui_counters.ts | 6 +- .../server/usage_counters/index.ts | 18 ++ .../usage_counters/saved_objects.test.ts | 69 +++++ .../server/usage_counters/saved_objects.ts | 85 ++++++ .../usage_counters/usage_counter.test.ts | 38 +++ .../server/usage_counters/usage_counter.ts | 48 +++ .../usage_counters_service.mock.ts | 40 +++ .../usage_counters_service.test.ts | 225 ++++++++++++++ .../usage_counters/usage_counters_service.ts | 178 ++++++++++++ .../telemetry/__fixtures__/ui_counters.ts | 8 + .../telemetry/__fixtures__/usage_counters.ts | 36 +++ .../apis/telemetry/telemetry_local.ts | 15 + .../apis/ui_counters/ui_counters.ts | 50 ++-- test/api_integration/config.js | 2 + .../saved_objects/ui_counters/data.json | 105 +++++++ .../saved_objects/ui_counters/data.json.gz | Bin 236 -> 0 bytes .../saved_objects/ui_counters/mappings.json | 7 + .../saved_objects/usage_counters/data.json | 74 +++++ .../usage_counters/mappings.json | 274 ++++++++++++++++++ test/plugin_functional/config.ts | 3 + .../plugins/usage_collection/kibana.json | 9 + .../plugins/usage_collection/package.json | 14 + .../plugins/usage_collection/server/config.ts | 18 ++ .../plugins/usage_collection/server/index.ts | 12 + .../plugins/usage_collection/server/plugin.ts | 44 +++ .../plugins/usage_collection/server/routes.ts | 24 ++ .../plugins/usage_collection/tsconfig.json | 18 ++ .../test_suites/usage_collection/index.ts | 15 + .../usage_collection/usage_counters.ts | 73 +++++ 57 files changed, 2713 insertions(+), 192 deletions(-) create mode 100644 src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/index.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_ui_counters_collector.test.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/index.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/register_rollups.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts create mode 100644 src/plugins/usage_collection/common/ui_counters.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/index.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/saved_objects.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counter.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts create mode 100644 src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts create mode 100644 test/api_integration/apis/telemetry/__fixtures__/usage_counters.ts create mode 100644 test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json delete mode 100644 test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json.gz create mode 100644 test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json create mode 100644 test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json create mode 100644 test/plugin_functional/plugins/usage_collection/kibana.json create mode 100644 test/plugin_functional/plugins/usage_collection/package.json create mode 100644 test/plugin_functional/plugins/usage_collection/server/config.ts create mode 100644 test/plugin_functional/plugins/usage_collection/server/index.ts create mode 100644 test/plugin_functional/plugins/usage_collection/server/plugin.ts create mode 100644 test/plugin_functional/plugins/usage_collection/server/routes.ts create mode 100644 test/plugin_functional/plugins/usage_collection/tsconfig.json create mode 100644 test/plugin_functional/test_suites/usage_collection/index.ts create mode 100644 test/plugin_functional/test_suites/usage_collection/usage_counters.ts diff --git a/src/plugins/kibana_usage_collection/server/collectors/index.ts b/src/plugins/kibana_usage_collection/server/collectors/index.ts index 10156b51ac183..41e9bb9f5886f 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/index.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/index.ts @@ -19,3 +19,7 @@ export { registerUiCounterSavedObjectType, registerUiCountersRollups, } from './ui_counters'; +export { + registerUsageCountersRollups, + registerUsageCountersUsageCollector, +} from './usage_counters'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts new file mode 100644 index 0000000000000..41932d7bc6ba3 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { UICounterSavedObject } from '../ui_counter_saved_object_type'; +export const rawUiCounters: UICounterSavedObject[] = [ + { + type: 'ui-counter', + id: 'Kibana_home:24112020:click:ingest_data_card_home_tutorial_directory', + attributes: { + count: 3, + }, + references: [], + updated_at: '2020-11-24T11:27:57.067Z', + version: 'WzI5LDFd', + }, + { + type: 'ui-counter', + id: 'Kibana_home:24112020:click:home_tutorial_directory', + attributes: { + count: 1, + }, + references: [], + updated_at: '2020-11-24T11:27:57.067Z', + version: 'WzI5NDRd', + }, + { + type: 'ui-counter', + id: 'Kibana_home:24112020:loaded:home_tutorial_directory', + attributes: { + count: 3, + }, + references: [], + updated_at: '2020-10-23T11:27:57.067Z', + version: 'WzI5NDRd', + }, +]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts new file mode 100644 index 0000000000000..9f7bfb6ff324f --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { UsageCountersSavedObject } from '../../../../../usage_collection/server'; + +export const rawUsageCounters: UsageCountersSavedObject[] = [ + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event', + attributes: { count: 1 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:17:57.693Z', + version: 'WzAsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'uiCounter:09042021:loaded:Kibana_home:home_tutorial_directory', + attributes: { count: 60 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2020-10-23T11:27:57.067Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event_4457914848544', + attributes: { count: 0 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event_malformed', + attributes: { count: 'malformed' }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'anotherDomainId:09042021:count:some_event_name', + attributes: { count: 4 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event_4457914848544_2', + attributes: { count: 8 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.031Z', + version: 'WzcsMV0=', + score: 0, + }, +]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts index 7e84bc852c9b5..7d25b18c17c5d 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts @@ -6,70 +6,136 @@ * Side Public License, v 1. */ -import { transformRawCounter } from './register_ui_counters_collector'; -import { UICounterSavedObject } from './ui_counter_saved_object_type'; +import { + transformRawUiCounterObject, + transformRawUsageCounterObject, + fetchUiCounters, +} from './register_ui_counters_collector'; +import { rawUiCounters } from './__fixtures__/ui_counter_saved_objects'; +import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects'; +import { savedObjectsClientMock } from '../../../../../core/server/mocks'; +import { UI_COUNTER_SAVED_OBJECT_TYPE } from './ui_counter_saved_object_type'; +import { USAGE_COUNTERS_SAVED_OBJECT_TYPE } from '../../../../usage_collection/server'; -describe('transformRawCounter', () => { - const mockRawUiCounters = [ - { - type: 'ui-counter', - id: 'Kibana_home:24112020:click:ingest_data_card_home_tutorial_directory', - attributes: { - count: 3, - }, - references: [], - updated_at: '2020-11-24T11:27:57.067Z', - version: 'WzI5LDFd', - }, - { - type: 'ui-counter', - id: 'Kibana_home:24112020:click:home_tutorial_directory', - attributes: { - count: 1, - }, - references: [], - updated_at: '2020-11-24T11:27:57.067Z', - version: 'WzI5NDRd', - }, - { - type: 'ui-counter', - id: 'Kibana_home:24112020:loaded:home_tutorial_directory', - attributes: { - count: 3, - }, - references: [], - updated_at: '2020-10-23T11:27:57.067Z', - version: 'WzI5NDRd', - }, - ] as UICounterSavedObject[]; +describe('transformRawUsageCounterObject', () => { + it('transforms usage counters savedObject raw entries', () => { + const result = rawUsageCounters.map(transformRawUsageCounterObject); + expect(result).toMatchInlineSnapshot(` + Array [ + Object { + "appName": "myApp", + "counterType": "count", + "eventName": "my_event", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:17:57.693Z", + "total": 1, + }, + Object { + "appName": "Kibana_home", + "counterType": "loaded", + "eventName": "home_tutorial_directory", + "fromTimestamp": "2020-10-23T00:00:00Z", + "lastUpdatedAt": "2020-10-23T11:27:57.067Z", + "total": 60, + }, + undefined, + undefined, + undefined, + Object { + "appName": "myApp", + "counterType": "count", + "eventName": "my_event_4457914848544_2", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.031Z", + "total": 8, + }, + ] + `); + }); +}); + +describe('transformRawUiCounterObject', () => { + it('transforms ui counters savedObject raw entries', () => { + const result = rawUiCounters.map(transformRawUiCounterObject); + expect(result).toMatchInlineSnapshot(` + Array [ + Object { + "appName": "Kibana_home", + "counterType": "click", + "eventName": "ingest_data_card_home_tutorial_directory", + "fromTimestamp": "2020-11-24T00:00:00Z", + "lastUpdatedAt": "2020-11-24T11:27:57.067Z", + "total": 3, + }, + Object { + "appName": "Kibana_home", + "counterType": "click", + "eventName": "home_tutorial_directory", + "fromTimestamp": "2020-11-24T00:00:00Z", + "lastUpdatedAt": "2020-11-24T11:27:57.067Z", + "total": 1, + }, + Object { + "appName": "Kibana_home", + "counterType": "loaded", + "eventName": "home_tutorial_directory", + "fromTimestamp": "2020-10-23T00:00:00Z", + "lastUpdatedAt": "2020-10-23T11:27:57.067Z", + "total": 3, + }, + ] + `); + }); +}); + +describe('fetchUiCounters', () => { + const soClientMock = savedObjectsClientMock.create(); + beforeEach(() => { + jest.clearAllMocks(); + }); + it('merges saved objects from both ui_counters and usage_counters saved objects', async () => { + soClientMock.find.mockImplementation(async ({ type }) => { + switch (type) { + case UI_COUNTER_SAVED_OBJECT_TYPE: + return { saved_objects: rawUiCounters }; + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: rawUsageCounters }; + default: + throw new Error(`unexpected type ${type}`); + } + }); + + const { dailyEvents } = await fetchUiCounters({ soClient: soClientMock }); + expect(dailyEvents).toHaveLength(6); + const intersectingEntry = dailyEvents.find( + (dailyEvent) => + dailyEvent.eventName === 'home_tutorial_directory' && + dailyEvent.fromTimestamp === '2020-10-23T00:00:00Z' + ); + const invalidCountEntry = dailyEvents.find( + (dailyEvent) => dailyEvent.eventName === 'my_event_malformed' + ); + + const zeroCountEntry = dailyEvents.find( + (dailyEvent) => dailyEvent.eventName === 'my_event_4457914848544' + ); + + const nonUiCountersEntry = dailyEvents.find( + (dailyEvent) => dailyEvent.eventName === 'some_event_name' + ); - it('transforms saved object raw entries', () => { - const result = mockRawUiCounters.map(transformRawCounter); - expect(result).toEqual([ - { - appName: 'Kibana_home', - eventName: 'ingest_data_card_home_tutorial_directory', - lastUpdatedAt: '2020-11-24T11:27:57.067Z', - fromTimestamp: '2020-11-24T00:00:00Z', - counterType: 'click', - total: 3, - }, - { - appName: 'Kibana_home', - eventName: 'home_tutorial_directory', - lastUpdatedAt: '2020-11-24T11:27:57.067Z', - fromTimestamp: '2020-11-24T00:00:00Z', - counterType: 'click', - total: 1, - }, - { - appName: 'Kibana_home', - eventName: 'home_tutorial_directory', - lastUpdatedAt: '2020-10-23T11:27:57.067Z', - fromTimestamp: '2020-10-23T00:00:00Z', - counterType: 'loaded', - total: 3, - }, - ]); + expect(invalidCountEntry).toBe(undefined); + expect(nonUiCountersEntry).toBe(undefined); + expect(zeroCountEntry).toBe(undefined); + expect(intersectingEntry).toMatchInlineSnapshot(` + Object { + "appName": "Kibana_home", + "counterType": "loaded", + "eventName": "home_tutorial_directory", + "fromTimestamp": "2020-10-23T00:00:00Z", + "lastUpdatedAt": "2020-10-23T11:27:57.067Z", + "total": 60, + } + `); }); }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts index dc3fac7382094..dbe35a35ebe94 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts @@ -7,13 +7,29 @@ */ import moment from 'moment'; -import { CollectorFetchContext, UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { chain } from 'lodash'; + import { UICounterSavedObject, UICounterSavedObjectAttributes, UI_COUNTER_SAVED_OBJECT_TYPE, } from './ui_counter_saved_object_type'; +import { + CollectorFetchContext, + UsageCollectionSetup, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + UsageCountersSavedObject, + UsageCountersSavedObjectAttributes, + deserializeCounterKey, + serializeCounterKey, +} from '../../../../usage_collection/server'; + +import { + deserializeUiCounterName, + serializeUiCounterName, +} from '../../../../usage_collection/common/ui_counters'; + interface UiCounterEvent { appName: string; eventName: string; @@ -27,12 +43,20 @@ export interface UiCountersUsage { dailyEvents: UiCounterEvent[]; } -export function transformRawCounter(rawUiCounter: UICounterSavedObject) { - const { id, attributes, updated_at: lastUpdatedAt } = rawUiCounter; +export function transformRawUiCounterObject( + rawUiCounter: UICounterSavedObject +): UiCounterEvent | undefined { + const { + id, + attributes: { count }, + updated_at: lastUpdatedAt, + } = rawUiCounter; + if (typeof count !== 'number' || count < 1) { + return; + } + const [appName, , counterType, ...restId] = id.split(':'); const eventName = restId.join(':'); - const counterTotal: unknown = attributes.count; - const total = typeof counterTotal === 'number' ? counterTotal : 0; const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); return { @@ -41,7 +65,101 @@ export function transformRawCounter(rawUiCounter: UICounterSavedObject) { lastUpdatedAt, fromTimestamp, counterType, - total, + total: count, + }; +} + +export function transformRawUsageCounterObject( + rawUsageCounter: UsageCountersSavedObject +): UiCounterEvent | undefined { + const { + id, + attributes: { count }, + updated_at: lastUpdatedAt, + } = rawUsageCounter; + const { counterName, counterType, domainId } = deserializeCounterKey(id); + + if (domainId !== 'uiCounter' || typeof count !== 'number' || count < 1) { + return; + } + + const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); + const { appName, eventName } = deserializeUiCounterName(counterName); + + return { + appName, + eventName, + lastUpdatedAt, + fromTimestamp, + counterType, + total: count, + }; +} + +export async function fetchUiCounters({ soClient }: CollectorFetchContext) { + const { + saved_objects: rawUsageCounters, + } = await soClient.find({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + fields: ['count'], + perPage: 10000, + }); + + const { saved_objects: rawUiCounters } = await soClient.find({ + type: UI_COUNTER_SAVED_OBJECT_TYPE, + fields: ['count'], + perPage: 10000, + }); + + const dailyEventsFromUiCounters = rawUiCounters.reduce((acc, raw) => { + try { + const event = transformRawUiCounterObject(raw); + if (event) { + const { appName, eventName, counterType } = event; + const key = serializeCounterKey({ + domainId: 'uiCounters', + counterName: serializeUiCounterName({ appName, eventName }), + counterType, + date: moment(event.fromTimestamp).format('DDMMYYYY'), + }); + + acc[key] = event; + } + } catch (_) { + // swallow error; allows sending successfully transformed objects. + } + return acc; + }, {} as Record); + + const dailyEventsFromUsageCounters = rawUsageCounters.reduce((acc, raw) => { + try { + const event = transformRawUsageCounterObject(raw); + if (event) { + acc[raw.id] = event; + } + } catch (_) { + // swallow error; allows sending successfully transformed objects. + } + return acc; + }, {} as Record); + + return { + dailyEvents: chain(dailyEventsFromUsageCounters) + .mergeWith( + dailyEventsFromUiCounters, + (value: UiCounterEvent | undefined, srcValue: UiCounterEvent): UiCounterEvent => { + if (!value) { + return srcValue; + } + + return { + ...srcValue, + total: srcValue.total + value.total, + }; + } + ) + .values() + .value(), }; } @@ -76,25 +194,7 @@ export function registerUiCountersUsageCollector(usageCollection: UsageCollectio }, }, }, - fetch: async ({ soClient }: CollectorFetchContext) => { - const { saved_objects: rawUiCounters } = await soClient.find({ - type: UI_COUNTER_SAVED_OBJECT_TYPE, - fields: ['count'], - perPage: 10000, - }); - - return { - dailyEvents: rawUiCounters.reduce((acc, raw) => { - try { - const aggEvent = transformRawCounter(raw); - acc.push(aggEvent); - } catch (_) { - // swallow error; allows sending successfully transformed objects. - } - return acc; - }, [] as UiCounterEvent[]), - }; - }, + fetch: fetchUiCounters, isReady: () => true, }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts index 9595101efb63b..85d31dd30576a 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts @@ -6,16 +6,21 @@ * Side Public License, v 1. */ -import { timer } from 'rxjs'; +import { Subject, timer } from 'rxjs'; +import { takeUntil } from 'rxjs/operators'; import { Logger, ISavedObjectsRepository } from 'kibana/server'; import { ROLL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants'; import { rollUiCounterIndices } from './rollups'; +export const stopRollingUiCounterIndicies$ = new Subject(); + export function registerUiCountersRollups( logger: Logger, getSavedObjectsClient: () => ISavedObjectsRepository | undefined ) { - timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL).subscribe(() => - rollUiCounterIndices(logger, getSavedObjectsClient()) - ); + timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL) + .pipe(takeUntil(stopRollingUiCounterIndicies$)) + .subscribe(() => + rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, getSavedObjectsClient()) + ); } diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts index 5cb91f7f898c1..4a6b55730a5df 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts @@ -7,9 +7,11 @@ */ import moment from 'moment'; +import * as Rx from 'rxjs'; import { isSavedObjectOlderThan, rollUiCounterIndices } from './rollups'; import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks'; import { SavedObjectsFindResult } from 'kibana/server'; + import { UICounterSavedObjectAttributes, UI_COUNTER_SAVED_OBJECT_TYPE, @@ -70,10 +72,12 @@ describe('isSavedObjectOlderThan', () => { describe('rollUiCounterIndices', () => { let logger: ReturnType; let savedObjectClient: ReturnType; + let stopRollingUiCounterIndicies$: Rx.Subject; beforeEach(() => { logger = loggingSystemMock.createLogger(); savedObjectClient = savedObjectsRepositoryMock.create(); + stopRollingUiCounterIndicies$ = new Rx.Subject(); }); it('returns undefined if no savedObjectsClient initialised yet', async () => { @@ -90,11 +94,27 @@ describe('rollUiCounterIndices', () => { throw new Error(`Unexpected type [${type}]`); } }); - await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toEqual([]); + await expect( + rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + ).resolves.toEqual([]); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).not.toBeCalled(); expect(logger.warn).toHaveBeenCalledTimes(0); }); + it('calls Subject complete() on empty saved objects', async () => { + savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { + switch (type) { + case UI_COUNTER_SAVED_OBJECT_TYPE: + return { saved_objects: [], total: 0, page, per_page: perPage }; + default: + throw new Error(`Unexpected type [${type}]`); + } + }); + await expect( + rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + ).resolves.toEqual([]); + expect(stopRollingUiCounterIndicies$.isStopped).toBe(true); + }); it(`deletes documents older than ${UI_COUNTERS_KEEP_DOCS_FOR_DAYS} days`, async () => { const mockSavedObjects = [ @@ -111,7 +131,9 @@ describe('rollUiCounterIndices', () => { throw new Error(`Unexpected type [${type}]`); } }); - await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toHaveLength(2); + await expect( + rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + ).resolves.toHaveLength(2); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).toHaveBeenCalledTimes(2); expect(savedObjectClient.delete).toHaveBeenNthCalledWith( @@ -131,7 +153,9 @@ describe('rollUiCounterIndices', () => { savedObjectClient.find.mockImplementation(async () => { throw new Error(`Expected error!`); }); - await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toEqual(undefined); + await expect( + rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + ).resolves.toEqual(undefined); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).not.toBeCalled(); expect(logger.warn).toHaveBeenCalledTimes(2); @@ -151,7 +175,9 @@ describe('rollUiCounterIndices', () => { savedObjectClient.delete.mockImplementation(async () => { throw new Error(`Expected error!`); }); - await expect(rollUiCounterIndices(logger, savedObjectClient)).resolves.toEqual(undefined); + await expect( + rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + ).resolves.toEqual(undefined); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).toHaveBeenCalledTimes(1); expect(savedObjectClient.delete).toHaveBeenNthCalledWith( diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts index 3a092f845c3a3..bd702a8381944 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts @@ -8,6 +8,7 @@ import { ISavedObjectsRepository, Logger } from 'kibana/server'; import moment from 'moment'; +import { Subject } from 'rxjs'; import { UI_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; import { @@ -38,6 +39,7 @@ export function isSavedObjectOlderThan({ export async function rollUiCounterIndices( logger: Logger, + stopRollingUiCounterIndicies$: Subject, savedObjectsClient?: ISavedObjectsRepository ) { if (!savedObjectsClient) { @@ -54,6 +56,20 @@ export async function rollUiCounterIndices( } ); + if (rawUiCounterDocs.length === 0) { + /** + * @deprecated 7.13 to be removed in 8.0.0 + * Stop triggering rollups when we've rolled up all documents. + * + * This Saved Object registry is no longer used. + * Migration from one SO registry to another is not yet supported. + * In a future release we can remove this piece of code and + * migrate any docs to the Usage Counters Saved object. + */ + + stopRollingUiCounterIndicies$.complete(); + } + const docsToDelete = rawUiCounterDocs.filter((doc) => isSavedObjectOlderThan({ numberOfDays: UI_COUNTERS_KEEP_DOCS_FOR_DAYS, diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts new file mode 100644 index 0000000000000..d8f0713edcb68 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { UsageCountersSavedObject } from '../../../../../usage_collection/server'; + +export const rawUsageCounters: UsageCountersSavedObject[] = [ + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:my_event_malformed', + attributes: { count: 13 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'anotherDomainId:09042021:count:some_event_name', + attributes: { count: 4 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'anotherDomainId:09042021:count:some_event_name', + attributes: { count: 4 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-11T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'anotherDomainId2:09042021:count:some_event_name', + attributes: { count: 1 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-20T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'anotherDomainId2:09042021:count:malformed_event', + attributes: { count: 'malformed' }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-20T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'anotherDomainId2:09042021:custom_type:some_event_name', + attributes: { count: 3 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-20T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, + { + type: 'usage-counters', + id: 'anotherDomainId3:09042021:custom_type:zero_count', + attributes: { count: 0 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-20T08:18:03.030Z', + version: 'WzUsMV0=', + score: 0, + }, +]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/index.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/index.ts new file mode 100644 index 0000000000000..1873fae42e54a --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { registerUsageCountersUsageCollector } from './register_usage_counters_collector'; +export { registerUsageCountersRollups } from './rollups'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_ui_counters_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_ui_counters_collector.test.ts new file mode 100644 index 0000000000000..945eb007fe23f --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_ui_counters_collector.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { transformRawCounter } from './register_usage_counters_collector'; +import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects'; + +describe('transformRawCounter', () => { + it('transforms saved object raw entries', () => { + const result = rawUsageCounters.map(transformRawCounter); + expect(result).toMatchInlineSnapshot(` + Array [ + undefined, + Object { + "counterName": "some_event_name", + "counterType": "count", + "domainId": "anotherDomainId", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.030Z", + "total": 4, + }, + Object { + "counterName": "some_event_name", + "counterType": "count", + "domainId": "anotherDomainId", + "fromTimestamp": "2021-04-11T00:00:00Z", + "lastUpdatedAt": "2021-04-11T08:18:03.030Z", + "total": 4, + }, + Object { + "counterName": "some_event_name", + "counterType": "count", + "domainId": "anotherDomainId2", + "fromTimestamp": "2021-04-20T00:00:00Z", + "lastUpdatedAt": "2021-04-20T08:18:03.030Z", + "total": 1, + }, + undefined, + Object { + "counterName": "some_event_name", + "counterType": "custom_type", + "domainId": "anotherDomainId2", + "fromTimestamp": "2021-04-20T00:00:00Z", + "lastUpdatedAt": "2021-04-20T08:18:03.030Z", + "total": 3, + }, + undefined, + ] + `); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts new file mode 100644 index 0000000000000..b6f5fc7f842eb --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment'; +import { + CollectorFetchContext, + UsageCollectionSetup, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + UsageCountersSavedObject, + UsageCountersSavedObjectAttributes, + deserializeCounterKey, +} from '../../../../usage_collection/server'; + +interface UsageCounterEvent { + domainId: string; + counterName: string; + counterType: string; + lastUpdatedAt?: string; + fromTimestamp?: string; + total: number; +} + +export interface UiCountersUsage { + dailyEvents: UsageCounterEvent[]; +} + +export function transformRawCounter( + rawUsageCounter: UsageCountersSavedObject +): UsageCounterEvent | undefined { + const { + id, + attributes: { count }, + updated_at: lastUpdatedAt, + } = rawUsageCounter; + const { domainId, counterType, counterName } = deserializeCounterKey(id); + const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); + + if (domainId === 'uiCounter' || typeof count !== 'number' || count < 1) { + return; + } + + return { + domainId, + counterName, + counterType, + lastUpdatedAt, + fromTimestamp, + total: count, + }; +} + +export function registerUsageCountersUsageCollector(usageCollection: UsageCollectionSetup) { + const collector = usageCollection.makeUsageCollector({ + type: 'usage_counters', + schema: { + dailyEvents: { + type: 'array', + items: { + domainId: { + type: 'keyword', + _meta: { description: 'Domain name of the metric (ie plugin name).' }, + }, + counterName: { + type: 'keyword', + _meta: { description: 'Name of the counter that happened.' }, + }, + lastUpdatedAt: { + type: 'date', + _meta: { description: 'Time at which the metric was last updated.' }, + }, + fromTimestamp: { + type: 'date', + _meta: { description: 'Time at which the metric was captured.' }, + }, + counterType: { + type: 'keyword', + _meta: { description: 'The type of counter used.' }, + }, + total: { + type: 'integer', + _meta: { description: 'The total number of times the event happened.' }, + }, + }, + }, + }, + fetch: async ({ soClient }: CollectorFetchContext) => { + const { + saved_objects: rawUsageCounters, + } = await soClient.find({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + fields: ['count'], + perPage: 10000, + }); + + return { + dailyEvents: rawUsageCounters.reduce((acc, rawUsageCounter) => { + try { + const event = transformRawCounter(rawUsageCounter); + if (event) { + acc.push(event); + } + return acc; + } catch (_) { + // swallow error; allows sending successfully transformed objects. + } + return acc; + }, [] as UsageCounterEvent[]), + }; + }, + isReady: () => true, + }); + + usageCollection.registerCollector(collector); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts new file mode 100644 index 0000000000000..b395d336a687d --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Roll indices every 24h + */ +export const ROLL_INDICES_INTERVAL = 24 * 60 * 60 * 1000; + +/** + * Start rolling indices after 5 minutes up + */ +export const ROLL_INDICES_START = 5 * 60 * 1000; + +/** + * Number of days to keep the Usage counters saved object documents + */ +export const USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS = 3; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/index.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/index.ts new file mode 100644 index 0000000000000..bf15f4d875860 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { registerUsageCountersRollups } from './register_rollups'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/register_rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/register_rollups.ts new file mode 100644 index 0000000000000..30ad993d54a8e --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/register_rollups.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { timer } from 'rxjs'; +import { Logger, ISavedObjectsRepository } from 'kibana/server'; +import { ROLL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants'; +import { rollUsageCountersIndices } from './rollups'; + +export function registerUsageCountersRollups( + logger: Logger, + getSavedObjectsClient: () => ISavedObjectsRepository | undefined +) { + timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL).subscribe(() => + rollUsageCountersIndices(logger, getSavedObjectsClient()) + ); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts new file mode 100644 index 0000000000000..967325f13d240 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment'; +import { isSavedObjectOlderThan, rollUsageCountersIndices } from './rollups'; +import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../../core/server/mocks'; +import { SavedObjectsFindResult } from '../../../../../../core/server'; + +import { + UsageCountersSavedObjectAttributes, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, +} from '../../../../../usage_collection/server'; + +import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; + +const createMockSavedObjectDoc = (updatedAt: moment.Moment, id: string) => + ({ + id, + type: 'ui-counter', + attributes: { + count: 3, + }, + references: [], + updated_at: updatedAt.format(), + version: 'WzI5LDFd', + score: 0, + } as SavedObjectsFindResult); + +describe('isSavedObjectOlderThan', () => { + it(`returns true if doc is older than x days`, () => { + const numberOfDays = 1; + const startDate = moment().format(); + const doc = createMockSavedObjectDoc(moment().subtract(2, 'days'), 'some-id'); + const result = isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, + }); + expect(result).toBe(true); + }); + + it(`returns false if doc is exactly x days old`, () => { + const numberOfDays = 1; + const startDate = moment().format(); + const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id'); + const result = isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, + }); + expect(result).toBe(false); + }); + + it(`returns false if doc is younger than x days`, () => { + const numberOfDays = 2; + const startDate = moment().format(); + const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id'); + const result = isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, + }); + expect(result).toBe(false); + }); +}); + +describe('rollUsageCountersIndices', () => { + let logger: ReturnType; + let savedObjectClient: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + savedObjectClient = savedObjectsRepositoryMock.create(); + }); + + it('returns undefined if no savedObjectsClient initialised yet', async () => { + await expect(rollUsageCountersIndices(logger, undefined)).resolves.toBe(undefined); + expect(logger.warn).toHaveBeenCalledTimes(0); + }); + + it('does not delete any documents on empty saved objects', async () => { + savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { + switch (type) { + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: [], total: 0, page, per_page: perPage }; + default: + throw new Error(`Unexpected type [${type}]`); + } + }); + await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toEqual([]); + expect(savedObjectClient.find).toBeCalled(); + expect(savedObjectClient.delete).not.toBeCalled(); + expect(logger.warn).toHaveBeenCalledTimes(0); + }); + + it(`deletes documents older than ${USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS} days`, async () => { + const mockSavedObjects = [ + createMockSavedObjectDoc(moment().subtract(5, 'days'), 'doc-id-1'), + createMockSavedObjectDoc(moment().subtract(1, 'days'), 'doc-id-2'), + createMockSavedObjectDoc(moment().subtract(6, 'days'), 'doc-id-3'), + ]; + + savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { + switch (type) { + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: mockSavedObjects, total: 0, page, per_page: perPage }; + default: + throw new Error(`Unexpected type [${type}]`); + } + }); + await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toHaveLength(2); + expect(savedObjectClient.find).toBeCalled(); + expect(savedObjectClient.delete).toHaveBeenCalledTimes(2); + expect(savedObjectClient.delete).toHaveBeenNthCalledWith( + 1, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + 'doc-id-1' + ); + expect(savedObjectClient.delete).toHaveBeenNthCalledWith( + 2, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + 'doc-id-3' + ); + expect(logger.warn).toHaveBeenCalledTimes(0); + }); + + it(`logs warnings on savedObject.find failure`, async () => { + savedObjectClient.find.mockImplementation(async () => { + throw new Error(`Expected error!`); + }); + await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toEqual(undefined); + expect(savedObjectClient.find).toBeCalled(); + expect(savedObjectClient.delete).not.toBeCalled(); + expect(logger.warn).toHaveBeenCalledTimes(2); + }); + + it(`logs warnings on savedObject.delete failure`, async () => { + const mockSavedObjects = [createMockSavedObjectDoc(moment().subtract(5, 'days'), 'doc-id-1')]; + + savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { + switch (type) { + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: mockSavedObjects, total: 0, page, per_page: perPage }; + default: + throw new Error(`Unexpected type [${type}]`); + } + }); + savedObjectClient.delete.mockImplementation(async () => { + throw new Error(`Expected error!`); + }); + await expect(rollUsageCountersIndices(logger, savedObjectClient)).resolves.toEqual(undefined); + expect(savedObjectClient.find).toBeCalled(); + expect(savedObjectClient.delete).toHaveBeenCalledTimes(1); + expect(savedObjectClient.delete).toHaveBeenNthCalledWith( + 1, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + 'doc-id-1' + ); + expect(logger.warn).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts new file mode 100644 index 0000000000000..74b555f1c296b --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ISavedObjectsRepository, Logger } from 'kibana/server'; +import moment from 'moment'; + +import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; + +import { + UsageCountersSavedObject, + USAGE_COUNTERS_SAVED_OBJECT_TYPE, +} from '../../../../../usage_collection/server'; + +export function isSavedObjectOlderThan({ + numberOfDays, + startDate, + doc, +}: { + numberOfDays: number; + startDate: moment.Moment | string | number; + doc: Pick; +}): boolean { + const { updated_at: updatedAt } = doc; + const today = moment(startDate).startOf('day'); + const updateDay = moment(updatedAt).startOf('day'); + + const diffInDays = today.diff(updateDay, 'days'); + if (diffInDays > numberOfDays) { + return true; + } + + return false; +} + +export async function rollUsageCountersIndices( + logger: Logger, + savedObjectsClient?: ISavedObjectsRepository +) { + if (!savedObjectsClient) { + return; + } + + const now = moment(); + + try { + const { + saved_objects: rawUiCounterDocs, + } = await savedObjectsClient.find({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + perPage: 1000, // Process 1000 at a time as a compromise of speed and overload + }); + + const docsToDelete = rawUiCounterDocs.filter((doc) => + isSavedObjectOlderThan({ + numberOfDays: USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS, + startDate: now, + doc, + }) + ); + + return await Promise.all( + docsToDelete.map(({ id }) => savedObjectsClient.delete(USAGE_COUNTERS_SAVED_OBJECT_TYPE, id)) + ); + } catch (err) { + logger.warn(`Failed to rollup Usage Counters saved objects.`); + logger.warn(err); + } +} diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index 5b903489e3ff3..06b75b214328c 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -34,6 +34,8 @@ import { registerUiCountersUsageCollector, registerUiCounterSavedObjectType, registerUiCountersRollups, + registerUsageCountersRollups, + registerUsageCountersUsageCollector, } from './collectors'; interface KibanaUsageCollectionPluginsDepsSetup { @@ -57,6 +59,8 @@ export class KibanaUsageCollectionPlugin implements Plugin { } public setup(coreSetup: CoreSetup, { usageCollection }: KibanaUsageCollectionPluginsDepsSetup) { + usageCollection.createUsageCounter('uiCounters'); + this.registerUsageCollectors( usageCollection, coreSetup, @@ -92,6 +96,9 @@ export class KibanaUsageCollectionPlugin implements Plugin { registerUiCountersRollups(this.logger.get('ui-counters'), getSavedObjectsClient); registerUiCountersUsageCollector(usageCollection); + registerUsageCountersRollups(this.logger.get('usage-counters-rollup'), getSavedObjectsClient); + registerUsageCountersUsageCollector(usageCollection); + registerOpsStatsCollector(usageCollection, metric$); registerKibanaUsageCollector(usageCollection, this.legacyConfig$); registerManagementUsageCollector(usageCollection, getUiSettingsClient); diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 05ac1eb84089d..30374fa8d3696 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -9274,6 +9274,53 @@ } } }, + "usage_counters": { + "properties": { + "dailyEvents": { + "type": "array", + "items": { + "properties": { + "domainId": { + "type": "keyword", + "_meta": { + "description": "Domain name of the metric (ie plugin name)." + } + }, + "counterName": { + "type": "keyword", + "_meta": { + "description": "Name of the counter that happened." + } + }, + "lastUpdatedAt": { + "type": "date", + "_meta": { + "description": "Time at which the metric was last updated." + } + }, + "fromTimestamp": { + "type": "date", + "_meta": { + "description": "Time at which the metric was captured." + } + }, + "counterType": { + "type": "keyword", + "_meta": { + "description": "The type of counter used." + } + }, + "total": { + "type": "integer", + "_meta": { + "description": "The total number of times the event happened." + } + } + } + } + } + } + }, "telemetry": { "properties": { "opt_in_status": { diff --git a/src/plugins/usage_collection/common/ui_counters.ts b/src/plugins/usage_collection/common/ui_counters.ts new file mode 100644 index 0000000000000..3ed6e44aee419 --- /dev/null +++ b/src/plugins/usage_collection/common/ui_counters.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const serializeUiCounterName = ({ + appName, + eventName, +}: { + appName: string; + eventName: string; +}) => { + return `${appName}:${eventName}`; +}; + +export const deserializeUiCounterName = (key: string) => { + const [appName, ...restKey] = key.split(':'); + const eventName = restKey.join(':'); + return { appName, eventName }; +}; diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index 32a58a6657eec..4de5691eaaa70 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -25,22 +25,6 @@ interface CollectorSetConfig { collectors?: AnyCollector[]; } -/** - * Public interface of the CollectorSet (makes it easier to mock only the public methods) - */ -export type CollectorSetPublic = Pick< - CollectorSet, - | 'makeStatsCollector' - | 'makeUsageCollector' - | 'registerCollector' - | 'getCollectorByType' - | 'areAllCollectorsReady' - | 'bulkFetch' - | 'bulkFetchUsage' - | 'toObject' - | 'toApiFieldNames' ->; - export class CollectorSet { private _waitingForAllCollectorsTimestamp?: number; private readonly logger: Logger; @@ -215,19 +199,19 @@ export class CollectorSet { * Convert an array of fetched stats results into key/object * @param statsData Array of fetched stats results */ - public toObject, T = unknown>( + public toObject = , T = unknown>( statsData: Array<{ type: string; result: T }> = [] - ): Result { + ): Result => { return Object.fromEntries(statsData.map(({ type, result }) => [type, result])) as Result; - } + }; /** * Rename fields to use API conventions * @param apiData Data to be normalized */ - public toApiFieldNames( + public toApiFieldNames = ( apiData: Record | unknown[] - ): Record | unknown[] { + ): Record | unknown[] => { // handle array and return early, or return a reduced object if (Array.isArray(apiData)) { return apiData.map((value) => this.getValueOrRecurse(value)); @@ -244,14 +228,14 @@ export class CollectorSet { return [newName, this.getValueOrRecurse(value)]; }) ); - } + }; - private getValueOrRecurse(value: unknown) { + private getValueOrRecurse = (value: unknown) => { if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { return this.toApiFieldNames(value as Record | unknown[]); // recurse } return value; - } + }; private makeCollectorSetFromArray = (collectors: AnyCollector[]) => { return new CollectorSet({ diff --git a/src/plugins/usage_collection/server/collector/index.ts b/src/plugins/usage_collection/server/collector/index.ts index 5f48f9fb93813..8462f31f654d4 100644 --- a/src/plugins/usage_collection/server/collector/index.ts +++ b/src/plugins/usage_collection/server/collector/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export { CollectorSet, CollectorSetPublic } from './collector_set'; +export { CollectorSet } from './collector_set'; export { Collector, AllowedSchemaTypes, diff --git a/src/plugins/usage_collection/server/config.ts b/src/plugins/usage_collection/server/config.ts index ff6ea8424ba61..a1b9a6165e1b5 100644 --- a/src/plugins/usage_collection/server/config.ts +++ b/src/plugins/usage_collection/server/config.ts @@ -11,6 +11,11 @@ import { PluginConfigDescriptor } from 'src/core/server'; import { DEFAULT_MAXIMUM_WAIT_TIME_FOR_ALL_COLLECTORS_IN_S } from '../common/constants'; export const configSchema = schema.object({ + usageCounters: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + retryCount: schema.number({ defaultValue: 1 }), + bufferDebounceMs: schema.number({ defaultValue: 30 * 1000 }), + }), uiCounters: schema.object({ enabled: schema.boolean({ defaultValue: true }), debug: schema.boolean({ defaultValue: schema.contextRef('dev') }), diff --git a/src/plugins/usage_collection/server/index.ts b/src/plugins/usage_collection/server/index.ts index dfc9d19b69646..81652e7df15b6 100644 --- a/src/plugins/usage_collection/server/index.ts +++ b/src/plugins/usage_collection/server/index.ts @@ -19,6 +19,18 @@ export { CollectorFetchContext, } from './collector'; +export type { + UsageCountersSavedObject, + UsageCountersSavedObjectAttributes, +} from './usage_counters'; + +export { + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + serializeCounterKey, + deserializeCounterKey, + UsageCounter, +} from './usage_counters'; + export { UsageCollectionSetup } from './plugin'; export { config } from './config'; export const plugin = (initializerContext: PluginInitializerContext) => diff --git a/src/plugins/usage_collection/server/plugin.ts b/src/plugins/usage_collection/server/plugin.ts index a44365ae9be9a..2685caad0dfd9 100644 --- a/src/plugins/usage_collection/server/plugin.ts +++ b/src/plugins/usage_collection/server/plugin.ts @@ -15,30 +15,78 @@ import { Plugin, } from 'src/core/server'; import { ConfigType } from './config'; -import { CollectorSet, CollectorSetPublic } from './collector'; +import { CollectorSet } from './collector'; import { setupRoutes } from './routes'; -export type UsageCollectionSetup = CollectorSetPublic; -export class UsageCollectionPlugin implements Plugin { +import { UsageCountersService } from './usage_counters'; +import type { UsageCountersServiceSetup } from './usage_counters'; + +export interface UsageCollectionSetup { + /** + * Creates and registers a usage counter to collect daily aggregated plugin counter events + */ + createUsageCounter: UsageCountersServiceSetup['createUsageCounter']; + /** + * Returns a usage counter by type + */ + getUsageCounterByType: UsageCountersServiceSetup['getUsageCounterByType']; + /** + * Creates a usage collector to collect plugin telemetry data. + * registerCollector must be called to connect the created collecter with the service. + */ + makeUsageCollector: CollectorSet['makeUsageCollector']; + /** + * Register a usage collector or a stats collector. + * Used to connect the created collector to telemetry. + */ + registerCollector: CollectorSet['registerCollector']; + /** + * Returns a usage collector by type + */ + getCollectorByType: CollectorSet['getCollectorByType']; + /* internal: telemetry use */ + areAllCollectorsReady: CollectorSet['areAllCollectorsReady']; + /* internal: telemetry use */ + bulkFetch: CollectorSet['bulkFetch']; + /* internal: telemetry use */ + toObject: CollectorSet['toObject']; + /* internal: monitoring use */ + toApiFieldNames: CollectorSet['toApiFieldNames']; + /* internal: telemtery and monitoring use */ + makeStatsCollector: CollectorSet['makeStatsCollector']; +} + +export class UsageCollectionPlugin implements Plugin { private readonly logger: Logger; private savedObjects?: ISavedObjectsRepository; + private usageCountersService?: UsageCountersService; + constructor(private readonly initializerContext: PluginInitializerContext) { this.logger = this.initializerContext.logger.get(); } - public setup(core: CoreSetup) { + public setup(core: CoreSetup): UsageCollectionSetup { const config = this.initializerContext.config.get(); const collectorSet = new CollectorSet({ - logger: this.logger.get('collector-set'), + logger: this.logger.get('usage-collection', 'collector-set'), maximumWaitTimeForAllCollectorsInS: config.maximumWaitTimeForAllCollectorsInS, }); - const globalConfig = this.initializerContext.config.legacy.get(); + this.usageCountersService = new UsageCountersService({ + logger: this.logger.get('usage-collection', 'usage-counters-service'), + retryCount: config.usageCounters.retryCount, + bufferDebounceMs: config.usageCounters.bufferDebounceMs, + }); + + const { createUsageCounter, getUsageCounterByType } = this.usageCountersService.setup(core); + const uiCountersUsageCounter = createUsageCounter('uiCounter'); + const globalConfig = this.initializerContext.config.legacy.get(); const router = core.http.createRouter(); setupRoutes({ router, + uiCountersUsageCounter, getSavedObjects: () => this.savedObjects, collectorSet, config: { @@ -52,15 +100,38 @@ export class UsageCollectionPlugin implements Plugin { overallStatus$: core.status.overall$, }); - return collectorSet; + return { + areAllCollectorsReady: collectorSet.areAllCollectorsReady, + bulkFetch: collectorSet.bulkFetch, + getCollectorByType: collectorSet.getCollectorByType, + makeStatsCollector: collectorSet.makeStatsCollector, + makeUsageCollector: collectorSet.makeUsageCollector, + registerCollector: collectorSet.registerCollector, + toApiFieldNames: collectorSet.toApiFieldNames, + toObject: collectorSet.toObject, + createUsageCounter, + getUsageCounterByType, + }; } public start({ savedObjects }: CoreStart) { this.logger.debug('Starting plugin'); + const config = this.initializerContext.config.get(); + if (!this.usageCountersService) { + throw new Error('plugin setup must be called first.'); + } + this.savedObjects = savedObjects.createInternalRepository(); + if (config.usageCounters.enabled) { + this.usageCountersService.start({ savedObjects }); + } else { + // call stop() to complete observers. + this.usageCountersService.stop(); + } } public stop() { this.logger.debug('Stopping plugin'); + this.usageCountersService?.stop(); } } diff --git a/src/plugins/usage_collection/server/report/store_report.test.ts b/src/plugins/usage_collection/server/report/store_report.test.ts index dfcdd1f8e7e42..5574220c7358d 100644 --- a/src/plugins/usage_collection/server/report/store_report.test.ts +++ b/src/plugins/usage_collection/server/report/store_report.test.ts @@ -12,9 +12,13 @@ import { savedObjectsRepositoryMock } from '../../../../core/server/mocks'; import { storeReport } from './store_report'; import { ReportSchemaType } from './schema'; import { METRIC_TYPE } from '@kbn/analytics'; +import { usageCountersServiceMock } from '../usage_counters/usage_counters_service.mock'; import moment from 'moment'; describe('store_report', () => { + const usageCountersServiceSetup = usageCountersServiceMock.createSetupContract(); + const uiCountersUsageCounter = usageCountersServiceSetup.createUsageCounter('uiCounter'); + const momentTimestamp = moment(); const date = momentTimestamp.format('DDMMYYYY'); @@ -64,34 +68,55 @@ describe('store_report', () => { }, }, }; - await storeReport(repository, report); + await storeReport(repository, uiCountersUsageCounter, report); - expect(repository.create).toHaveBeenCalledWith( - 'ui-metric', - { count: 1 }, - { - id: 'key-user-agent:test-user-agent', - overwrite: true, - } - ); - expect(repository.incrementCounter).toHaveBeenNthCalledWith( - 1, - 'ui-metric', - 'test-app-name:test-event-name', - [{ fieldName: 'count', incrementBy: 3 }] - ); - expect(repository.incrementCounter).toHaveBeenNthCalledWith( - 2, - 'ui-counter', - `test-app-name:${date}:${METRIC_TYPE.LOADED}:test-event-name`, - [{ fieldName: 'count', incrementBy: 1 }] - ); - expect(repository.incrementCounter).toHaveBeenNthCalledWith( - 3, - 'ui-counter', - `test-app-name:${date}:${METRIC_TYPE.CLICK}:test-event-name`, - [{ fieldName: 'count', incrementBy: 2 }] - ); + expect(repository.create.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "ui-metric", + Object { + "count": 1, + }, + Object { + "id": "key-user-agent:test-user-agent", + "overwrite": true, + }, + ], + ] + `); + + expect(repository.incrementCounter.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "ui-metric", + "test-app-name:test-event-name", + Array [ + Object { + "fieldName": "count", + "incrementBy": 3, + }, + ], + ], + ] + `); + expect(uiCountersUsageCounter.incrementCounter.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "counterName": "test-app-name:test-event-name", + "counterType": "loaded", + "incrementBy": 1, + }, + ], + Array [ + Object { + "counterName": "test-app-name:test-event-name", + "counterType": "click", + "incrementBy": 2, + }, + ], + ] + `); expect(storeApplicationUsageMock).toHaveBeenCalledTimes(1); expect(storeApplicationUsageMock).toHaveBeenCalledWith( @@ -108,7 +133,7 @@ describe('store_report', () => { uiCounter: void 0, application_usage: void 0, }; - await storeReport(repository, report); + await storeReport(repository, uiCountersUsageCounter, report); expect(repository.bulkCreate).not.toHaveBeenCalled(); expect(repository.incrementCounter).not.toHaveBeenCalled(); diff --git a/src/plugins/usage_collection/server/report/store_report.ts b/src/plugins/usage_collection/server/report/store_report.ts index 0545a54792d45..1647fb8893be1 100644 --- a/src/plugins/usage_collection/server/report/store_report.ts +++ b/src/plugins/usage_collection/server/report/store_report.ts @@ -11,9 +11,12 @@ import moment from 'moment'; import { chain, sumBy } from 'lodash'; import { ReportSchemaType } from './schema'; import { storeApplicationUsage } from './store_application_usage'; +import { UsageCounter } from '../usage_counters'; +import { serializeUiCounterName } from '../../common/ui_counters'; export async function storeReport( internalRepository: ISavedObjectsRepository, + uiCountersUsageCounter: UsageCounter, report: ReportSchemaType ) { const uiCounters = report.uiCounter ? Object.entries(report.uiCounter) : []; @@ -21,7 +24,6 @@ export async function storeReport( const appUsages = report.application_usage ? Object.values(report.application_usage) : []; const momentTimestamp = moment(); - const date = momentTimestamp.format('DDMMYYYY'); const timestamp = momentTimestamp.toDate(); return Promise.allSettled([ @@ -55,14 +57,14 @@ export async function storeReport( }) .value(), // UI Counters - ...uiCounters.map(async ([key, metric]) => { + ...uiCounters.map(async ([, metric]) => { const { appName, eventName, total, type } = metric; - const savedObjectId = `${appName}:${date}:${type}:${eventName}`; - return [ - await internalRepository.incrementCounter('ui-counter', savedObjectId, [ - { fieldName: 'count', incrementBy: total }, - ]), - ]; + const counterName = serializeUiCounterName({ appName, eventName }); + uiCountersUsageCounter.incrementCounter({ + counterName, + counterType: type, + incrementBy: total, + }); }), // Application Usage storeApplicationUsage(internalRepository, appUsages, timestamp), diff --git a/src/plugins/usage_collection/server/routes/index.ts b/src/plugins/usage_collection/server/routes/index.ts index 0e17ebcbfd695..3591ca2d59f9c 100644 --- a/src/plugins/usage_collection/server/routes/index.ts +++ b/src/plugins/usage_collection/server/routes/index.ts @@ -16,14 +16,16 @@ import { Observable } from 'rxjs'; import { CollectorSet } from '../collector'; import { registerUiCountersRoute } from './ui_counters'; import { registerStatsRoute } from './stats'; - +import { UsageCounter } from '../usage_counters'; export function setupRoutes({ router, + uiCountersUsageCounter, getSavedObjects, ...rest }: { router: IRouter; getSavedObjects: () => ISavedObjectsRepository | undefined; + uiCountersUsageCounter: UsageCounter; config: { allowAnonymous: boolean; kibanaIndex: string; @@ -39,6 +41,6 @@ export function setupRoutes({ metrics: MetricsServiceSetup; overallStatus$: Observable; }) { - registerUiCountersRoute(router, getSavedObjects); + registerUiCountersRoute(router, getSavedObjects, uiCountersUsageCounter); registerStatsRoute({ router, ...rest }); } diff --git a/src/plugins/usage_collection/server/routes/ui_counters.ts b/src/plugins/usage_collection/server/routes/ui_counters.ts index 07983ba1d65ca..c03541b1032b6 100644 --- a/src/plugins/usage_collection/server/routes/ui_counters.ts +++ b/src/plugins/usage_collection/server/routes/ui_counters.ts @@ -9,10 +9,12 @@ import { schema } from '@kbn/config-schema'; import { IRouter, ISavedObjectsRepository } from 'src/core/server'; import { storeReport, reportSchema } from '../report'; +import { UsageCounter } from '../usage_counters'; export function registerUiCountersRoute( router: IRouter, - getSavedObjects: () => ISavedObjectsRepository | undefined + getSavedObjects: () => ISavedObjectsRepository | undefined, + uiCountersUsageCounter: UsageCounter ) { router.post( { @@ -30,7 +32,7 @@ export function registerUiCountersRoute( if (!internalRepository) { throw Error(`The saved objects client hasn't been initialised yet`); } - await storeReport(internalRepository, report); + await storeReport(internalRepository, uiCountersUsageCounter, report); return res.ok({ body: { status: 'ok' } }); } catch (error) { return res.ok({ body: { status: 'fail' } }); diff --git a/src/plugins/usage_collection/server/usage_counters/index.ts b/src/plugins/usage_collection/server/usage_counters/index.ts new file mode 100644 index 0000000000000..12e2d27edef9f --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export type { UsageCountersServiceSetup } from './usage_counters_service'; +export type { UsageCountersSavedObjectAttributes, UsageCountersSavedObject } from './saved_objects'; + +export { UsageCountersService } from './usage_counters_service'; +export { UsageCounter } from './usage_counter'; +export { + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + serializeCounterKey, + deserializeCounterKey, +} from './saved_objects'; diff --git a/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts new file mode 100644 index 0000000000000..5992674ec0363 --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { deserializeCounterKey, serializeCounterKey, storeCounter } from './saved_objects'; +import { savedObjectsRepositoryMock } from '../../../../core/server/mocks'; +import { CounterMetric } from './usage_counter'; +import moment from 'moment'; + +describe('counterKey', () => { + test('#serializeCounterKey returns a serialized string', () => { + const result = serializeCounterKey({ + domainId: 'a', + counterName: 'b', + counterType: 'c', + date: moment('09042021', 'DDMMYYYY'), + }); + + expect(result).toMatchInlineSnapshot(`"a:09042021:c:b"`); + }); + + test('#deserializeCounterKey', () => { + const key = deserializeCounterKey('a:09042021:c:b'); + expect(key).toMatchInlineSnapshot(` + Object { + "counterName": "b", + "counterType": "c", + "domainId": "a", + } + `); + }); +}); + +describe('storeCounter', () => { + const internalRepository = savedObjectsRepositoryMock.create(); + + afterAll(() => { + jest.resetAllMocks(); + }); + + it('stores counter in a saved object', async () => { + const counterMetric: CounterMetric = { + domainId: 'a', + counterName: 'b', + counterType: 'c', + incrementBy: 13, + }; + + await storeCounter(counterMetric, internalRepository); + + expect(internalRepository.incrementCounter).toBeCalledTimes(1); + expect(internalRepository.incrementCounter.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "usage-counters", + "a:09042021:c:b", + Array [ + Object { + "fieldName": "count", + "incrementBy": 13, + }, + ], + ] + `); + }); +}); diff --git a/src/plugins/usage_collection/server/usage_counters/saved_objects.ts b/src/plugins/usage_collection/server/usage_counters/saved_objects.ts new file mode 100644 index 0000000000000..d95b5b1fdd748 --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/saved_objects.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + SavedObject, + SavedObjectsRepository, + SavedObjectAttributes, + SavedObjectsServiceSetup, +} from 'kibana/server'; +import moment from 'moment'; +import { CounterMetric } from './usage_counter'; + +export interface UsageCountersSavedObjectAttributes extends SavedObjectAttributes { + count: number; +} + +export type UsageCountersSavedObject = SavedObject; + +export const USAGE_COUNTERS_SAVED_OBJECT_TYPE = 'usage-counters'; + +export const registerUsageCountersSavedObjectType = ( + savedObjectsSetup: SavedObjectsServiceSetup +) => { + savedObjectsSetup.registerType({ + name: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + hidden: false, + namespaceType: 'agnostic', + mappings: { + properties: { + count: { type: 'integer' }, + }, + }, + }); +}; + +export interface SerializeCounterParams { + domainId: string; + counterName: string; + counterType: string; + date: moment.MomentInput; +} + +export const serializeCounterKey = ({ + domainId, + counterName, + counterType, + date, +}: SerializeCounterParams) => { + const dayDate = moment(date).format('DDMMYYYY'); + return `${domainId}:${dayDate}:${counterType}:${counterName}`; +}; + +export const deserializeCounterKey = (key: string) => { + const [domainId, , counterType, ...restKey] = key.split(':'); + const counterName = restKey.join(':'); + + return { + domainId, + counterName, + counterType, + }; +}; + +export const storeCounter = async ( + counterMetric: CounterMetric, + internalRepository: Pick +) => { + const { counterName, counterType, domainId, incrementBy } = counterMetric; + + const key = serializeCounterKey({ + date: moment.now(), + domainId, + counterName, + counterType, + }); + + return await internalRepository.incrementCounter(USAGE_COUNTERS_SAVED_OBJECT_TYPE, key, [ + { fieldName: 'count', incrementBy }, + ]); +}; diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts b/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts new file mode 100644 index 0000000000000..3602ff1a29376 --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counter.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { UsageCounter, CounterMetric } from './usage_counter'; +import * as Rx from 'rxjs'; +import * as rxOp from 'rxjs/operators'; + +describe('UsageCounter', () => { + const domainId = 'test-domain-id'; + const counter$ = new Rx.Subject(); + const usageCounter = new UsageCounter({ domainId, counter$ }); + + afterAll(() => { + counter$.complete(); + }); + + describe('#incrementCounter', () => { + it('#incrementCounter calls counter$.next', async () => { + const result = counter$.pipe(rxOp.take(1), rxOp.toArray()).toPromise(); + usageCounter.incrementCounter({ counterName: 'test', counterType: 'type', incrementBy: 13 }); + await expect(result).resolves.toEqual([ + { counterName: 'test', counterType: 'type', domainId: 'test-domain-id', incrementBy: 13 }, + ]); + }); + + it('passes default configs to counter$', async () => { + const result = counter$.pipe(rxOp.take(1), rxOp.toArray()).toPromise(); + usageCounter.incrementCounter({ counterName: 'test' }); + await expect(result).resolves.toEqual([ + { counterName: 'test', counterType: 'count', domainId: 'test-domain-id', incrementBy: 1 }, + ]); + }); + }); +}); diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counter.ts b/src/plugins/usage_collection/server/usage_counters/usage_counter.ts new file mode 100644 index 0000000000000..78cbc7554ab1f --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counter.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Rx from 'rxjs'; + +export interface CounterMetric { + domainId: string; + counterName: string; + counterType: string; + incrementBy: number; +} + +export interface UsageCounterDeps { + domainId: string; + counter$: Rx.Subject; +} + +export interface IncrementCounterConfig { + counterName: string; + counterType?: string; + incrementBy?: number; +} + +export class UsageCounter { + private domainId: string; + private counter$: Rx.Subject; + + constructor({ domainId, counter$ }: UsageCounterDeps) { + this.domainId = domainId; + this.counter$ = counter$; + } + + public incrementCounter = (config: IncrementCounterConfig) => { + const { counterName, counterType = 'count', incrementBy = 1 } = config; + + this.counter$.next({ + counterName, + domainId: this.domainId, + counterType, + incrementBy, + }); + }; +} diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts new file mode 100644 index 0000000000000..beb67d1eb2607 --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { PublicMethodsOf } from '@kbn/utility-types'; +import type { UsageCountersService, UsageCountersServiceSetup } from './usage_counters_service'; +import type { UsageCounter } from './usage_counter'; + +const createSetupContractMock = () => { + const setupContract: jest.Mocked = { + createUsageCounter: jest.fn(), + getUsageCounterByType: jest.fn(), + }; + + setupContract.createUsageCounter.mockReturnValue(({ + incrementCounter: jest.fn(), + } as unknown) as jest.Mocked); + + return setupContract; +}; + +const createUsageCountersServiceMock = () => { + const mocked: jest.Mocked> = { + setup: jest.fn(), + start: jest.fn(), + stop: jest.fn(), + }; + + mocked.setup.mockReturnValue(createSetupContractMock()); + return mocked; +}; + +export const usageCountersServiceMock = { + create: createUsageCountersServiceMock, + createSetupContract: createSetupContractMock, +}; diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts new file mode 100644 index 0000000000000..d119a13b292d0 --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { UsageCountersService } from './usage_counters_service'; +import { loggingSystemMock, coreMock } from '../../../../core/server/mocks'; +import * as rxOp from 'rxjs/operators'; +import moment from 'moment'; + +const tick = () => { + jest.useRealTimers(); + return new Promise((resolve) => setTimeout(resolve, 1)); +}; + +describe('UsageCountersService', () => { + const retryCount = 1; + const bufferDebounceMs = 0; + const mockNow = 1617954426939; + const logger = loggingSystemMock.createLogger(); + const coreSetup = coreMock.createSetup(); + const coreStart = coreMock.createStart(); + + beforeEach(() => { + jest.spyOn(moment, 'now').mockReturnValue(mockNow); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('stores data in cache during setup', async () => { + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDebounceMs }); + const { createUsageCounter } = usageCountersService.setup(coreSetup); + + const usageCounter = createUsageCounter('test-counter'); + + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA' }); + + const dataInSourcePromise = usageCountersService.source$.pipe(rxOp.toArray()).toPromise(); + usageCountersService.flushCache$.next(); + usageCountersService.source$.complete(); + await expect(dataInSourcePromise).resolves.toHaveLength(2); + }); + + it('registers savedObject type during setup', () => { + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDebounceMs }); + const { createUsageCounter } = usageCountersService.setup(coreSetup); + expect(coreSetup.savedObjects.registerType).toBeCalledTimes(1); + }); + + it('flushes cached data on start', async () => { + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDebounceMs }); + + const mockRepository = coreStart.savedObjects.createInternalRepository(); + const mockIncrementCounter = jest.fn(); + mockRepository.incrementCounter = mockIncrementCounter; + + coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository); + const { createUsageCounter } = usageCountersService.setup(coreSetup); + + const usageCounter = createUsageCounter('test-counter'); + + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA' }); + + const dataInSourcePromise = usageCountersService.source$.pipe(rxOp.toArray()).toPromise(); + usageCountersService.start(coreStart); + usageCountersService.source$.complete(); + + await expect(dataInSourcePromise).resolves.toMatchInlineSnapshot(` + Array [ + Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + "incrementBy": 1, + }, + Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + "incrementBy": 1, + }, + ] + `); + }); + + it('buffers data into savedObject', async () => { + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDebounceMs }); + + const mockRepository = coreStart.savedObjects.createInternalRepository(); + const mockIncrementCounter = jest.fn().mockResolvedValue('success'); + mockRepository.incrementCounter = mockIncrementCounter; + + coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository); + const { createUsageCounter } = usageCountersService.setup(coreSetup); + jest.useFakeTimers('modern'); + const usageCounter = createUsageCounter('test-counter'); + + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA' }); + + usageCountersService.start(coreStart); + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterB' }); + jest.runOnlyPendingTimers(); + expect(mockIncrementCounter).toBeCalledTimes(2); + expect(mockIncrementCounter.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "usage-counters", + "test-counter:09042021:count:counterA", + Array [ + Object { + "fieldName": "count", + "incrementBy": 3, + }, + ], + ], + Array [ + "usage-counters", + "test-counter:09042021:count:counterB", + Array [ + Object { + "fieldName": "count", + "incrementBy": 1, + }, + ], + ], + ] + `); + }); + + it('retries errors by `retryCount` times before failing to store', async () => { + const usageCountersService = new UsageCountersService({ + logger, + retryCount: 1, + bufferDebounceMs, + }); + + const mockRepository = coreStart.savedObjects.createInternalRepository(); + const mockError = new Error('failed.'); + const mockIncrementCounter = jest.fn().mockImplementation((_, key) => { + switch (key) { + case 'test-counter:09042021:count:counterA': + throw mockError; + case 'test-counter:09042021:count:counterB': + return 'pass'; + default: + throw new Error(`unknown key ${key}`); + } + }); + + mockRepository.incrementCounter = mockIncrementCounter; + + coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository); + const { createUsageCounter } = usageCountersService.setup(coreSetup); + jest.useFakeTimers('modern'); + const usageCounter = createUsageCounter('test-counter'); + + usageCountersService.start(coreStart); + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterB' }); + jest.runOnlyPendingTimers(); + + // wait for retries to kick in on next scheduler call + await tick(); + // number of incrementCounter calls + number of retries + expect(mockIncrementCounter).toBeCalledTimes(2 + 1); + expect(logger.debug).toHaveBeenNthCalledWith(1, 'Store counters into savedObjects', [ + mockError, + 'pass', + ]); + }); + + it('buffers counters within `bufferDebounceMs` time', async () => { + const usageCountersService = new UsageCountersService({ + logger, + retryCount, + bufferDebounceMs: 30000, + }); + + const mockRepository = coreStart.savedObjects.createInternalRepository(); + const mockIncrementCounter = jest.fn().mockImplementation((_data, key, counter) => { + expect(counter).toHaveLength(1); + return { key, incrementBy: counter[0].incrementBy }; + }); + + mockRepository.incrementCounter = mockIncrementCounter; + + coreStart.savedObjects.createInternalRepository.mockReturnValue(mockRepository); + + const { createUsageCounter } = usageCountersService.setup(coreSetup); + jest.useFakeTimers('modern'); + const usageCounter = createUsageCounter('test-counter'); + + usageCountersService.start(coreStart); + usageCounter.incrementCounter({ counterName: 'counterA' }); + usageCounter.incrementCounter({ counterName: 'counterA' }); + jest.advanceTimersByTime(30000); + + usageCounter.incrementCounter({ counterName: 'counterA' }); + jest.runOnlyPendingTimers(); + + // wait for debounce to kick in on next scheduler call + await tick(); + expect(mockIncrementCounter).toBeCalledTimes(2); + expect(mockIncrementCounter.mock.results.map(({ value }) => value)).toMatchInlineSnapshot(` + Array [ + Object { + "incrementBy": 2, + "key": "test-counter:09042021:count:counterA", + }, + Object { + "incrementBy": 1, + "key": "test-counter:09042021:count:counterA", + }, + ] + `); + }); +}); diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts new file mode 100644 index 0000000000000..d0616caf600fe --- /dev/null +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Rx from 'rxjs'; +import * as rxOp from 'rxjs/operators'; +import { + SavedObjectsRepository, + SavedObjectsServiceSetup, + SavedObjectsServiceStart, +} from 'src/core/server'; +import type { Logger } from 'src/core/server'; + +import moment from 'moment'; +import { CounterMetric, UsageCounter } from './usage_counter'; +import { + registerUsageCountersSavedObjectType, + storeCounter, + serializeCounterKey, +} from './saved_objects'; + +export interface UsageCountersServiceDeps { + logger: Logger; + retryCount: number; + bufferDebounceMs: number; +} + +export interface UsageCountersServiceSetup { + createUsageCounter: (type: string) => UsageCounter; + getUsageCounterByType: (type: string) => UsageCounter | undefined; +} + +/* internal */ +export interface UsageCountersServiceSetupDeps { + savedObjects: SavedObjectsServiceSetup; +} + +/* internal */ +export interface UsageCountersServiceStartDeps { + savedObjects: SavedObjectsServiceStart; +} + +export class UsageCountersService { + private readonly stop$ = new Rx.Subject(); + private readonly retryCount: number; + private readonly bufferDebounceMs: number; + + private readonly counterSets = new Map(); + private readonly source$ = new Rx.Subject(); + private readonly counter$ = this.source$.pipe(rxOp.multicast(new Rx.Subject()), rxOp.refCount()); + private readonly flushCache$ = new Rx.Subject(); + + private readonly stopCaching$ = new Rx.Subject(); + + private readonly logger: Logger; + + constructor({ logger, retryCount, bufferDebounceMs }: UsageCountersServiceDeps) { + this.logger = logger; + this.retryCount = retryCount; + this.bufferDebounceMs = bufferDebounceMs; + } + + public setup = (core: UsageCountersServiceSetupDeps): UsageCountersServiceSetup => { + const cache$ = new Rx.ReplaySubject(); + const storingCache$ = new Rx.BehaviorSubject(false); + // flush cache data from cache -> source + this.flushCache$ + .pipe( + rxOp.exhaustMap(() => cache$), + rxOp.takeUntil(this.stop$) + ) + .subscribe((data) => { + storingCache$.next(true); + this.source$.next(data); + }); + + // store data into cache when not paused + storingCache$ + .pipe( + rxOp.distinctUntilChanged(), + rxOp.switchMap((isStoring) => (isStoring ? Rx.EMPTY : this.source$)), + rxOp.takeUntil(Rx.merge(this.stopCaching$, this.stop$)) + ) + .subscribe((data) => { + cache$.next(data); + storingCache$.next(false); + }); + + registerUsageCountersSavedObjectType(core.savedObjects); + + return { + createUsageCounter: this.createUsageCounter, + getUsageCounterByType: this.getUsageCounterByType, + }; + }; + + public start = ({ savedObjects }: UsageCountersServiceStartDeps): void => { + this.stopCaching$.next(); + const internalRepository = savedObjects.createInternalRepository(); + this.counter$ + .pipe( + rxOp.buffer(this.source$.pipe(rxOp.debounceTime(this.bufferDebounceMs))), + rxOp.map((counters) => Object.values(this.mergeCounters(counters))), + rxOp.takeUntil(this.stop$), + rxOp.concatMap((counters) => this.storeDate$(counters, internalRepository)) + ) + .subscribe((results) => { + this.logger.debug('Store counters into savedObjects', results); + }); + + this.flushCache$.next(); + }; + + public stop = () => { + this.stop$.next(); + }; + + private storeDate$( + counters: CounterMetric[], + internalRepository: Pick + ) { + return Rx.forkJoin( + counters.map((counter) => + Rx.defer(() => storeCounter(counter, internalRepository)).pipe( + rxOp.retry(this.retryCount), + rxOp.catchError((error) => { + this.logger.warn(error); + return Rx.of(error); + }) + ) + ) + ); + } + + private createUsageCounter = (type: string): UsageCounter => { + if (this.counterSets.get(type)) { + throw new Error(`Usage counter set "${type}" already exists.`); + } + + const counterSet = new UsageCounter({ + domainId: type, + counter$: this.source$, + }); + + this.counterSets.set(type, counterSet); + + return counterSet; + }; + + private getUsageCounterByType = (type: string): UsageCounter | undefined => { + return this.counterSets.get(type); + }; + + private mergeCounters = (counters: CounterMetric[]): Record => { + const date = moment.now(); + return counters.reduce((acc, counter) => { + const { counterName, domainId, counterType } = counter; + const key = serializeCounterKey({ domainId, counterName, counterType, date }); + const existingCounter = acc[key]; + if (!existingCounter) { + acc[key] = counter; + return acc; + } + return { + ...acc, + [key]: { + ...existingCounter, + ...counter, + incrementBy: existingCounter.incrementBy + counter.incrementBy, + }, + }; + }, {} as Record); + }; +} diff --git a/test/api_integration/apis/telemetry/__fixtures__/ui_counters.ts b/test/api_integration/apis/telemetry/__fixtures__/ui_counters.ts index 762b917918202..07a11f3876d86 100644 --- a/test/api_integration/apis/telemetry/__fixtures__/ui_counters.ts +++ b/test/api_integration/apis/telemetry/__fixtures__/ui_counters.ts @@ -8,6 +8,14 @@ export const basicUiCounters = { dailyEvents: [ + { + appName: 'myApp', + eventName: 'some_app_event', + lastUpdatedAt: '2021-11-20T11:43:00.961Z', + fromTimestamp: '2021-11-20T00:00:00Z', + counterType: 'count', + total: 2, + }, { appName: 'myApp', eventName: 'my_event_885082425109579', diff --git a/test/api_integration/apis/telemetry/__fixtures__/usage_counters.ts b/test/api_integration/apis/telemetry/__fixtures__/usage_counters.ts new file mode 100644 index 0000000000000..988bc2e77528d --- /dev/null +++ b/test/api_integration/apis/telemetry/__fixtures__/usage_counters.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const basicUsageCounters = { + dailyEvents: [ + { + domainId: 'anotherDomainId', + counterName: 'some_event_name', + counterType: 'count', + lastUpdatedAt: '2021-11-20T11:43:00.961Z', + fromTimestamp: '2021-11-20T00:00:00Z', + total: 3, + }, + { + domainId: 'anotherDomainId', + counterName: 'some_event_name', + counterType: 'count', + lastUpdatedAt: '2021-04-09T11:43:00.961Z', + fromTimestamp: '2021-04-09T00:00:00Z', + total: 2, + }, + { + domainId: 'anotherDomainId2', + counterName: 'some_event_name', + counterType: 'count', + lastUpdatedAt: '2021-04-20T08:18:03.030Z', + fromTimestamp: '2021-04-20T00:00:00Z', + total: 1, + }, + ], +}; diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts index d0a09ee58d335..9b92576c84b3a 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/test/api_integration/apis/telemetry/telemetry_local.ts @@ -9,6 +9,7 @@ import expect from '@kbn/expect'; import supertestAsPromised from 'supertest-as-promised'; import { basicUiCounters } from './__fixtures__/ui_counters'; +import { basicUsageCounters } from './__fixtures__/usage_counters'; import type { FtrProviderContext } from '../../ftr_provider_context'; import type { SavedObject } from '../../../../src/core/server'; import ossRootTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_root.json'; @@ -153,6 +154,20 @@ export default function ({ getService }: FtrProviderContext) { }); }); + describe('Usage Counters telemetry', () => { + before('Add UI Counters saved objects', () => + esArchiver.load('saved_objects/usage_counters') + ); + after('cleanup saved objects changes', () => + esArchiver.unload('saved_objects/usage_counters') + ); + + it('returns usage counters aggregated by day', async () => { + const stats = await retrieveTelemetry(supertest); + expect(stats.stack_stats.kibana.plugins.usage_counters).to.eql(basicUsageCounters); + }); + }); + describe('application usage limits', () => { function createSavedObject(viewId?: string) { return supertest diff --git a/test/api_integration/apis/ui_counters/ui_counters.ts b/test/api_integration/apis/ui_counters/ui_counters.ts index 2d55e224f31ce..aa201eb6a96ff 100644 --- a/test/api_integration/apis/ui_counters/ui_counters.ts +++ b/test/api_integration/apis/ui_counters/ui_counters.ts @@ -7,11 +7,10 @@ */ import expect from '@kbn/expect'; -import { ReportManager, METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics'; +import { ReportManager, METRIC_TYPE, UiCounterMetricType, Report } from '@kbn/analytics'; import moment from 'moment'; import { FtrProviderContext } from '../../ftr_provider_context'; -import { SavedObject } from '../../../../src/core/server'; -import { UICounterSavedObjectAttributes } from '../../../../src/plugins/kibana_usage_collection/server/collectors/ui_counters/ui_counter_saved_object_type'; +import { UsageCountersSavedObject } from '../../../../src/plugins/usage_collection/server'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -24,10 +23,22 @@ export default function ({ getService }: FtrProviderContext) { count, }); + const sendReport = async (report: Report) => { + await supertest + .post('/api/ui_counters/_report') + .set('kbn-xsrf', 'kibana') + .set('content-type', 'application/json') + .send({ report }) + .expect(200); + + // wait for SO to index data into ES + await new Promise((res) => setTimeout(res, 5 * 1000)); + }; + const getCounterById = ( - savedObjects: Array>, + savedObjects: UsageCountersSavedObject[], targetId: string - ): SavedObject => { + ): UsageCountersSavedObject => { const savedObject = savedObjects.find(({ id }: { id: string }) => id === targetId); if (!savedObject) { throw new Error(`Unable to find savedObject id ${targetId}`); @@ -40,30 +51,25 @@ export default function ({ getService }: FtrProviderContext) { const dayDate = moment().format('DDMMYYYY'); before(async () => await esArchiver.emptyKibanaIndex()); - it('stores ui counter events in savedObjects', async () => { + it('stores ui counter events in usage counters savedObjects', async () => { const reportManager = new ReportManager(); const { report } = reportManager.assignReports([ createUiCounterEvent('my_event', METRIC_TYPE.COUNT), ]); - await supertest - .post('/api/ui_counters/_report') - .set('kbn-xsrf', 'kibana') - .set('content-type', 'application/json') - .send({ report }) - .expect(200); + await sendReport(report); const { body: { saved_objects: savedObjects }, } = await supertest - .get('/api/saved_objects/_find?type=ui-counter') + .get('/api/saved_objects/_find?type=usage-counters') .set('kbn-xsrf', 'kibana') .expect(200); const countTypeEvent = getCounterById( savedObjects, - `myApp:${dayDate}:${METRIC_TYPE.COUNT}:my_event` + `uiCounter:${dayDate}:${METRIC_TYPE.COUNT}:myApp:my_event` ); expect(countTypeEvent.attributes.count).to.eql(1); }); @@ -78,35 +84,31 @@ export default function ({ getService }: FtrProviderContext) { createUiCounterEvent(`${uniqueEventName}_2`, METRIC_TYPE.COUNT), createUiCounterEvent(uniqueEventName, METRIC_TYPE.CLICK, 2), ]); - await supertest - .post('/api/ui_counters/_report') - .set('kbn-xsrf', 'kibana') - .set('content-type', 'application/json') - .send({ report }) - .expect(200); + + await sendReport(report); const { body: { saved_objects: savedObjects }, } = await supertest - .get('/api/saved_objects/_find?type=ui-counter&fields=count') + .get('/api/saved_objects/_find?type=usage-counters&fields=count') .set('kbn-xsrf', 'kibana') .expect(200); const countTypeEvent = getCounterById( savedObjects, - `myApp:${dayDate}:${METRIC_TYPE.COUNT}:${uniqueEventName}` + `uiCounter:${dayDate}:${METRIC_TYPE.COUNT}:myApp:${uniqueEventName}` ); expect(countTypeEvent.attributes.count).to.eql(1); const clickTypeEvent = getCounterById( savedObjects, - `myApp:${dayDate}:${METRIC_TYPE.CLICK}:${uniqueEventName}` + `uiCounter:${dayDate}:${METRIC_TYPE.CLICK}:myApp:${uniqueEventName}` ); expect(clickTypeEvent.attributes.count).to.eql(2); const secondEvent = getCounterById( savedObjects, - `myApp:${dayDate}:${METRIC_TYPE.COUNT}:${uniqueEventName}_2` + `uiCounter:${dayDate}:${METRIC_TYPE.COUNT}:myApp:${uniqueEventName}_2` ); expect(secondEvent.attributes.count).to.eql(1); }); diff --git a/test/api_integration/config.js b/test/api_integration/config.js index 1c19dd24fa96b..e9ebe7cb0e82c 100644 --- a/test/api_integration/config.js +++ b/test/api_integration/config.js @@ -31,6 +31,8 @@ export default async function ({ readConfigFile }) { '--server.xsrf.disableProtection=true', '--server.compression.referrerWhitelist=["some-host.com"]', `--savedObjects.maxImportExportSize=10001`, + // for testing set debounce to 0 to immediately flush counters into saved objects. + '--usageCollection.usageCounters.bufferDebounceMs=0', ], }, }; diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json new file mode 100644 index 0000000000000..3f80ca64ff4bd --- /dev/null +++ b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json @@ -0,0 +1,105 @@ +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:loaded:my_event_885082425109579", + "source": { + "ui-counter": { + "count": 1 + }, + "type": "ui-counter", + "updated_at": "2020-11-30T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:count:my_event_885082425109579_2", + "source": { + "ui-counter": { + "count": 1 + }, + "type": "ui-counter", + "updated_at": "2020-11-30T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:count:my_event_885082425109579_2", + "source": { + "ui-counter": { + "count": 1 + }, + "type": "ui-counter", + "updated_at": "2020-10-28T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:click:my_event_885082425109579", + "source": { + "ui-counter": { + "count": 2 + }, + "type": "ui-counter", + "updated_at": "2020-11-30T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "ui-counter:myApp:30112020:click:my_event_885082425109579", + "source": { + "ui-counter": { + "count": 2 + }, + "type": "ui-counter", + "updated_at": "2020-11-30T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "uiCounter:09042021:count:myApp:some_app_event", + "source": { + "usage-counters": { + "count": 2 + }, + "type": "usage-counters", + "updated_at": "2021-11-20T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId:09042021:count:some_event_name", + "source": { + "usage-counters": { + "count": 2 + }, + "type": "usage-counters", + "updated_at": "2021-11-20T11:43:00.961Z" + } + } +} + diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json.gz b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json.gz deleted file mode 100644 index 3f42c777260b3bb8c9892f0b4e7c1ed0f18292ed..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 236 zcmVQOZ*BnXld%qhFc5}!o`Q6yXaMqJu++`}+TP?Von^e4lhfqX_qj)CCC~=tX558Es+9vX<)Z1mUGT zi(1Sg$EAa&q=hzhr&@j;4o$-&KxDvxS6WCVEzMQ0>Ml>y1X32W1R+cI+0y2wOfof+Hf2BMuN|J3NtDK6!3Uo;Pk8 m%#1(glCys@znBbAmVPmrsw^%W{3W*ei+KQ7tJo%F1ONd3YHSDq diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json index 926fd5d79faa0..261e228261c64 100644 --- a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json +++ b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json @@ -35,6 +35,13 @@ } } }, + "usage-counters": { + "properties": { + "count": { + "type": "integer" + } + } + }, "dashboard": { "properties": { "description": { diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json new file mode 100644 index 0000000000000..53be9c9fc995c --- /dev/null +++ b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json @@ -0,0 +1,74 @@ +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "uiCounter:20112020:count:myApp:some_app_event", + "source": { + "usage-counters": { + "count": 2 + }, + "type": "usage-counters", + "updated_at": "2021-11-20T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId:20112020:count:some_event_name", + "source": { + "usage-counters": { + "count": 3 + }, + "type": "usage-counters", + "updated_at": "2021-11-20T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId:09042021:count:some_event_name", + "source": { + "usage-counters": { + "count": 2 + }, + "type": "usage-counters", + "updated_at": "2021-04-09T11:43:00.961Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId2:09042021:count:some_event_name", + "source": { + "usage-counters": { + "count": 1 + }, + "type": "usage-counters", + "updated_at": "2021-04-20T08:18:03.030Z" + } + } +} + +{ + "type": "doc", + "value": { + "index": ".kibana", + "id": "anotherDomainId3:09042021:custom_type:zero_count", + "source": { + "usage-counters": { + "count": 0 + }, + "type": "usage-counters", + "updated_at": "2021-04-20T08:18:03.030Z" + } + } +} diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json new file mode 100644 index 0000000000000..f89299a7899f4 --- /dev/null +++ b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json @@ -0,0 +1,274 @@ +{ + "type": "index", + "value": { + "index": ".kibana", + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + }, + "mappings": { + "dynamic": "strict", + "properties": { + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + }, + "defaultIndex": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "usage-counters": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "index-pattern": { + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + } + } + }, + "search": { + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "server": { + "properties": { + "uuid": { + "type": "keyword" + } + } + }, + "timelion-sheet": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "namespace": { + "type": "keyword" + }, + "references": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "type": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "url": { + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 2048 + } + } + } + } + }, + "visualization": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchId": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + } + } + } + } +} diff --git a/test/plugin_functional/config.ts b/test/plugin_functional/config.ts index 1651e213ee82d..c51de3b674647 100644 --- a/test/plugin_functional/config.ts +++ b/test/plugin_functional/config.ts @@ -31,6 +31,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('./test_suites/data_plugin'), require.resolve('./test_suites/saved_objects_management'), require.resolve('./test_suites/saved_objects_hidden_type'), + require.resolve('./test_suites/usage_collection'), ], services: { ...functionalConfig.get('services'), @@ -59,6 +60,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--corePluginDeprecations.oldProperty=hello', '--corePluginDeprecations.secret=100', '--corePluginDeprecations.noLongerUsed=still_using', + // for testing set debounce to 0 to immediately flush counters into saved objects. + '--usageCollection.usageCounters.bufferDebounceMs=0', ...plugins.map( (pluginDir) => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}` ), diff --git a/test/plugin_functional/plugins/usage_collection/kibana.json b/test/plugin_functional/plugins/usage_collection/kibana.json new file mode 100644 index 0000000000000..d0cbc63834fec --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/kibana.json @@ -0,0 +1,9 @@ +{ + "id": "usageCollectionTestPlugin", + "version": "0.0.1", + "kibanaVersion": "kibana", + "configPath": ["usageCollectionTestPlugin"], + "optionalPlugins": ["usageCollection"], + "server": true, + "ui": false +} diff --git a/test/plugin_functional/plugins/usage_collection/package.json b/test/plugin_functional/plugins/usage_collection/package.json new file mode 100644 index 0000000000000..33289bd8d727f --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/package.json @@ -0,0 +1,14 @@ +{ + "name": "usage_collection_test_plugin", + "version": "1.0.0", + "main": "target/test/plugin_functional/plugins/usage_collection", + "kibana": { + "version": "kibana", + "templateVersion": "1.0.0" + }, + "license": "SSPL-1.0 OR Elastic License 2.0", + "scripts": { + "kbn": "node ../../../../scripts/kbn.js", + "build": "rm -rf './target' && ../../../../node_modules/.bin/tsc" + } +} diff --git a/test/plugin_functional/plugins/usage_collection/server/config.ts b/test/plugin_functional/plugins/usage_collection/server/config.ts new file mode 100644 index 0000000000000..f79cb1fcbb74f --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/server/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor } from 'kibana/server'; + +const configSchema = schema.object({}); + +type ConfigType = TypeOf; + +export const config: PluginConfigDescriptor = { + schema: configSchema, +}; diff --git a/test/plugin_functional/plugins/usage_collection/server/index.ts b/test/plugin_functional/plugins/usage_collection/server/index.ts new file mode 100644 index 0000000000000..e26e5efcc88e5 --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/server/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { UsageCollectionTestPlugin } from './plugin'; + +export { config } from './config'; +export const plugin = () => new UsageCollectionTestPlugin(); diff --git a/test/plugin_functional/plugins/usage_collection/server/plugin.ts b/test/plugin_functional/plugins/usage_collection/server/plugin.ts new file mode 100644 index 0000000000000..0ce60bd5c21e3 --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/server/plugin.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Plugin, CoreSetup } from 'kibana/server'; +import { + UsageCollectionSetup, + UsageCounter, +} from '../../../../../src/plugins/usage_collection/server'; +import { registerRoutes } from './routes'; + +export interface TestPluginDepsSetup { + usageCollection?: UsageCollectionSetup; +} + +export class UsageCollectionTestPlugin implements Plugin { + private usageCounter?: UsageCounter; + + public setup(core: CoreSetup, { usageCollection }: TestPluginDepsSetup) { + if (!usageCollection) { + throw new Error('Optional plugin `usageCollection` is expected to be enabled for testing.'); + } + + const usageCounter = usageCollection?.createUsageCounter('usageCollectionTestPlugin'); + + registerRoutes(core.http, usageCounter); + usageCounter.incrementCounter({ + counterName: 'duringSetup', + incrementBy: 10, + }); + usageCounter.incrementCounter({ counterName: 'duringSetup' }); + this.usageCounter = usageCounter; + } + + public start() { + this.usageCounter?.incrementCounter({ counterName: 'duringStart' }); + } + + public stop() {} +} diff --git a/test/plugin_functional/plugins/usage_collection/server/routes.ts b/test/plugin_functional/plugins/usage_collection/server/routes.ts new file mode 100644 index 0000000000000..264d1b5cfbfca --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/server/routes.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { HttpServiceSetup } from 'kibana/server'; +import { UsageCounter } from '../../../../../src/plugins/usage_collection/server'; + +export function registerRoutes(http: HttpServiceSetup, usageCounter: UsageCounter) { + const router = http.createRouter(); + router.post( + { + path: '/api/usage_collection_test_plugin/', + validate: false, + }, + async (context, req, res) => { + usageCounter.incrementCounter({ counterName: 'routeAccessed' }); + return res.ok(); + } + ); +} diff --git a/test/plugin_functional/plugins/usage_collection/tsconfig.json b/test/plugin_functional/plugins/usage_collection/tsconfig.json new file mode 100644 index 0000000000000..3d9d8ca9451d4 --- /dev/null +++ b/test/plugin_functional/plugins/usage_collection/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./target", + "skipLibCheck": true + }, + "include": [ + "index.ts", + "public/**/*.ts", + "public/**/*.tsx", + "server/**/*.ts", + "../../../../typings/**/*", + ], + "exclude": [], + "references": [ + { "path": "../../../../src/core/tsconfig.json" } + ] +} diff --git a/test/plugin_functional/test_suites/usage_collection/index.ts b/test/plugin_functional/test_suites/usage_collection/index.ts new file mode 100644 index 0000000000000..201b7b04ff222 --- /dev/null +++ b/test/plugin_functional/test_suites/usage_collection/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PluginFunctionalProviderContext } from '../../services'; + +export default function ({ loadTestFile }: PluginFunctionalProviderContext) { + describe('usage collection', function () { + loadTestFile(require.resolve('./usage_counters')); + }); +} diff --git a/test/plugin_functional/test_suites/usage_collection/usage_counters.ts b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts new file mode 100644 index 0000000000000..444c0edbac5e2 --- /dev/null +++ b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { PluginFunctionalProviderContext } from '../../services'; +import { + UsageCountersSavedObject, + deserializeCounterKey, + serializeCounterKey, +} from '../../../../src/plugins/usage_collection/server/usage_counters'; + +export default function ({ getService, getPageObjects }: PluginFunctionalProviderContext) { + const supertest = getService('supertest'); + + async function getSavedObjectCounters() { + return await supertest + .get('/api/saved_objects/_find?type=usage-counters') + .set('kbn-xsrf', 'true') + .expect(200) + .then(({ body }) => { + expect(body.total).to.above(1); + return (body.saved_objects as UsageCountersSavedObject[]).reduce((acc, savedObj) => { + const { counterName, domainId } = deserializeCounterKey(savedObj.id); + + if (domainId === 'usageCollectionTestPlugin') { + const { count } = savedObj.attributes; + acc[counterName] = count; + } + + return acc; + }, {} as Record); + }); + } + + describe('Usage Counters service', () => { + before(async () => { + const key = serializeCounterKey({ + counterName: 'routeAccessed', + counterType: 'count', + domainId: 'usageCollectionTestPlugin', + date: Date.now(), + }); + + await supertest.delete(`/api/saved_objects/usage-counters/${key}`).set('kbn-xsrf', 'true'); + }); + + it('stores usage counters sent during start and setup', async () => { + const { duringSetup, duringStart, routeAccessed } = await getSavedObjectCounters(); + + expect(duringSetup).to.be(11); + expect(duringStart).to.be(1); + expect(routeAccessed).to.be(undefined); + }); + + it('stores usage counters triggered by runtime activities', async () => { + await supertest + .post('/api/usage_collection_test_plugin/') + .set('kbn-xsrf', 'true') + .expect(200); + + // wait until ES indexes the counter SavedObject; + await new Promise((res) => setTimeout(res, 7 * 1000)); + + const { routeAccessed } = await getSavedObjectCounters(); + expect(routeAccessed).to.be(1); + }); + }); +} From c2c9c89058053d119e9b7862b47a3096e3e15689 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Sun, 11 Apr 2021 11:15:56 +0300 Subject: [PATCH 02/11] code review changes --- .../server/__snapshots__/index.test.ts.snap | 19 ------ ...emetry_application_usage_collector.test.ts | 2 +- .../server/collectors/core/index.test.ts | 2 +- .../server/collectors/kibana/index.test.ts | 2 +- .../telemetry_management_collector.test.ts | 2 +- .../server/collectors/ops_stats/index.test.ts | 2 +- .../__fixtures__/ui_counter_saved_objects.ts | 10 +++ .../usage_counter_saved_objects.ts | 21 +++--- .../register_ui_counters_collector.test.ts | 62 +++++++++++++++--- .../collectors/ui_counters/rollups/rollups.ts | 2 +- .../server/collectors/ui_metric/index.test.ts | 2 +- .../usage_counter_saved_objects.ts | 15 +---- ...register_usage_counters_collector.test.ts} | 0 .../usage_counters/rollups/constants.ts | 2 +- .../usage_counters/rollups/rollups.test.ts | 3 +- .../server/{index.test.ts => plugin.test.ts} | 60 +++++++++++++++-- src/plugins/usage_collection/server/config.ts | 2 +- src/plugins/usage_collection/server/mocks.ts | 64 +++++++++++++++---- .../server/report/store_report.test.ts | 7 +- .../usage_collection/server/routes/index.ts | 2 +- .../server/usage_collection.mock.ts | 58 ----------------- .../usage_counters/saved_objects.test.ts | 6 ++ .../usage_counters_service.test.ts | 14 ++-- .../register_usage_collector.test.ts | 6 +- .../register_timeseries_collector.test.ts | 2 +- .../register_vega_collector.test.ts | 2 +- .../register_visualizations_collector.test.ts | 2 +- 27 files changed, 214 insertions(+), 157 deletions(-) delete mode 100644 src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap rename src/plugins/kibana_usage_collection/server/collectors/usage_counters/{register_ui_counters_collector.test.ts => register_usage_counters_collector.test.ts} (100%) rename src/plugins/kibana_usage_collection/server/{index.test.ts => plugin.test.ts} (59%) delete mode 100644 src/plugins/usage_collection/server/usage_collection.mock.ts diff --git a/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap b/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap deleted file mode 100644 index 2180d6a0fcc4e..0000000000000 --- a/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap +++ /dev/null @@ -1,19 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`kibana_usage_collection Runs the setup method without issues 1`] = `true`; - -exports[`kibana_usage_collection Runs the setup method without issues 2`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 3`] = `true`; - -exports[`kibana_usage_collection Runs the setup method without issues 4`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 5`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 6`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 7`] = `true`; - -exports[`kibana_usage_collection Runs the setup method without issues 8`] = `false`; - -exports[`kibana_usage_collection Runs the setup method without issues 9`] = `true`; diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts index f1b21af5506e6..da4e1b101914f 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.test.ts @@ -10,7 +10,7 @@ import { savedObjectsRepositoryMock, loggingSystemMock } from '../../../../../co import { Collector, createUsageCollectionSetupMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { MAIN_APP_DEFAULT_VIEW_ID } from '../../../../usage_collection/common/constants'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts index 4409442f4c70a..cbc38129fdddf 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts @@ -9,7 +9,7 @@ import { Collector, createUsageCollectionSetupMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { registerCoreUsageCollector } from '.'; import { coreUsageDataServiceMock, loggingSystemMock } from '../../../../../core/server/mocks'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts index 1d0329cb01d69..e1afbfbcecc4e 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/kibana/index.test.ts @@ -15,7 +15,7 @@ import { Collector, createCollectorFetchContextMock, createUsageCollectionSetupMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { registerKibanaUsageCollector } from './'; const logger = loggingSystemMock.createLogger(); diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.test.ts index a8ac778226082..cb0b1c045397d 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.test.ts @@ -11,7 +11,7 @@ import { Collector, createUsageCollectionSetupMock, createCollectorFetchContextMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { registerManagementUsageCollector, diff --git a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts index a90197e7a25ab..dfd6a93b7ea18 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ops_stats/index.test.ts @@ -11,7 +11,7 @@ import { Collector, createUsageCollectionSetupMock, createCollectorFetchContextMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { registerOpsStatsCollector } from './'; import { OpsMetrics } from '../../../../../core/server'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts index 41932d7bc6ba3..e592e897c5861 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts @@ -38,4 +38,14 @@ export const rawUiCounters: UICounterSavedObject[] = [ updated_at: '2020-10-23T11:27:57.067Z', version: 'WzI5NDRd', }, + { + type: 'ui-counter', + id: 'Kibana_home:24112020:click:only_reported_in_ui_counters', + attributes: { + count: 1, + }, + references: [], + updated_at: '2020-11-24T11:27:57.067Z', + version: 'WzI5NDRd', + }, ]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts index 9f7bfb6ff324f..b828bb68153c4 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts @@ -16,8 +16,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:17:57.693Z', - version: 'WzAsMV0=', - score: 0, }, { type: 'usage-counters', @@ -26,8 +24,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2020-10-23T11:27:57.067Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', @@ -36,18 +32,15 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', id: 'uiCounter:09042021:count:myApp:my_event_malformed', + // @ts-expect-error attributes: { count: 'malformed' }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', @@ -56,8 +49,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', @@ -66,7 +57,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.031Z', - version: 'WzcsMV0=', - score: 0, + }, + { + type: 'usage-counters', + id: 'uiCounter:09042021:count:myApp:only_reported_in_usage_counters', + attributes: { count: 1 }, + references: [], + coreMigrationVersion: '8.0.0', + updated_at: '2021-04-09T08:18:03.031Z', }, ]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts index 7d25b18c17c5d..70e6733bb6b0e 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts @@ -49,6 +49,14 @@ describe('transformRawUsageCounterObject', () => { "lastUpdatedAt": "2021-04-09T08:18:03.031Z", "total": 8, }, + Object { + "appName": "myApp", + "counterType": "count", + "eventName": "only_reported_in_usage_counters", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.031Z", + "total": 1, + }, ] `); }); @@ -83,6 +91,14 @@ describe('transformRawUiCounterObject', () => { "lastUpdatedAt": "2020-10-23T11:27:57.067Z", "total": 3, }, + Object { + "appName": "Kibana_home", + "counterType": "click", + "eventName": "only_reported_in_ui_counters", + "fromTimestamp": "2020-11-24T00:00:00Z", + "lastUpdatedAt": "2020-11-24T11:27:57.067Z", + "total": 1, + }, ] `); }); @@ -94,6 +110,7 @@ describe('fetchUiCounters', () => { jest.clearAllMocks(); }); it('merges saved objects from both ui_counters and usage_counters saved objects', async () => { + // @ts-expect-error incomplete mock implementation soClientMock.find.mockImplementation(async ({ type }) => { switch (type) { case UI_COUNTER_SAVED_OBJECT_TYPE: @@ -105,28 +122,55 @@ describe('fetchUiCounters', () => { } }); + // @ts-expect-error incomplete mock implementation const { dailyEvents } = await fetchUiCounters({ soClient: soClientMock }); - expect(dailyEvents).toHaveLength(6); + expect(dailyEvents).toHaveLength(8); const intersectingEntry = dailyEvents.find( - (dailyEvent) => - dailyEvent.eventName === 'home_tutorial_directory' && - dailyEvent.fromTimestamp === '2020-10-23T00:00:00Z' + ({ eventName, fromTimestamp }) => + eventName === 'home_tutorial_directory' && fromTimestamp === '2020-10-23T00:00:00Z' ); + + const onlyFromUICountersEntry = dailyEvents.find( + ({ eventName }) => eventName === 'only_reported_in_ui_counters' + ); + + const onlyFromUsageCountersEntry = dailyEvents.find( + ({ eventName }) => eventName === 'only_reported_in_usage_counters' + ); + const invalidCountEntry = dailyEvents.find( - (dailyEvent) => dailyEvent.eventName === 'my_event_malformed' + ({ eventName }) => eventName === 'my_event_malformed' ); const zeroCountEntry = dailyEvents.find( - (dailyEvent) => dailyEvent.eventName === 'my_event_4457914848544' + ({ eventName }) => eventName === 'my_event_4457914848544' ); - const nonUiCountersEntry = dailyEvents.find( - (dailyEvent) => dailyEvent.eventName === 'some_event_name' - ); + const nonUiCountersEntry = dailyEvents.find(({ eventName }) => eventName === 'some_event_name'); expect(invalidCountEntry).toBe(undefined); expect(nonUiCountersEntry).toBe(undefined); expect(zeroCountEntry).toBe(undefined); + expect(onlyFromUICountersEntry).toMatchInlineSnapshot(` + Object { + "appName": "Kibana_home", + "counterType": "click", + "eventName": "only_reported_in_ui_counters", + "fromTimestamp": "2020-11-24T00:00:00Z", + "lastUpdatedAt": "2020-11-24T11:27:57.067Z", + "total": 1, + } + `); + expect(onlyFromUsageCountersEntry).toMatchInlineSnapshot(` + Object { + "appName": "myApp", + "counterType": "count", + "eventName": "only_reported_in_usage_counters", + "fromTimestamp": "2021-04-09T00:00:00Z", + "lastUpdatedAt": "2021-04-09T08:18:03.031Z", + "total": 1, + } + `); expect(intersectingEntry).toMatchInlineSnapshot(` Object { "appName": "Kibana_home", diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts index bd702a8381944..19174a648838d 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts @@ -8,7 +8,7 @@ import { ISavedObjectsRepository, Logger } from 'kibana/server'; import moment from 'moment'; -import { Subject } from 'rxjs'; +import type { Subject } from 'rxjs'; import { UI_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; import { diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts index 77413cc7d7d9d..51ecbf736bfc1 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_metric/index.test.ts @@ -11,7 +11,7 @@ import { Collector, createUsageCollectionSetupMock, createCollectorFetchContextMock, -} from '../../../../usage_collection/server/usage_collection.mock'; +} from '../../../../usage_collection/server/mocks'; import { registerUiMetricUsageCollector } from './'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts index d8f0713edcb68..99f23bb08de23 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts @@ -16,8 +16,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', @@ -26,8 +24,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', @@ -36,8 +32,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-11T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', @@ -46,18 +40,15 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-20T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', id: 'anotherDomainId2:09042021:count:malformed_event', + // @ts-expect-error attributes: { count: 'malformed' }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-20T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', @@ -66,8 +57,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-20T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, { type: 'usage-counters', @@ -76,7 +65,5 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-20T08:18:03.030Z', - version: 'WzUsMV0=', - score: 0, }, ]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_ui_counters_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts similarity index 100% rename from src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_ui_counters_collector.test.ts rename to src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.test.ts diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts index b395d336a687d..1c1ca3f466df2 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/constants.ts @@ -19,4 +19,4 @@ export const ROLL_INDICES_START = 5 * 60 * 1000; /** * Number of days to keep the Usage counters saved object documents */ -export const USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS = 3; +export const USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS = 5; diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts index 967325f13d240..e13dab5ca72b0 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts @@ -101,6 +101,7 @@ describe('rollUsageCountersIndices', () => { it(`deletes documents older than ${USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS} days`, async () => { const mockSavedObjects = [ createMockSavedObjectDoc(moment().subtract(5, 'days'), 'doc-id-1'), + createMockSavedObjectDoc(moment().subtract(9, 'days'), 'doc-id-1'), createMockSavedObjectDoc(moment().subtract(1, 'days'), 'doc-id-2'), createMockSavedObjectDoc(moment().subtract(6, 'days'), 'doc-id-3'), ]; @@ -140,7 +141,7 @@ describe('rollUsageCountersIndices', () => { }); it(`logs warnings on savedObject.delete failure`, async () => { - const mockSavedObjects = [createMockSavedObjectDoc(moment().subtract(5, 'days'), 'doc-id-1')]; + const mockSavedObjects = [createMockSavedObjectDoc(moment().subtract(7, 'days'), 'doc-id-1')]; savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => { switch (type) { diff --git a/src/plugins/kibana_usage_collection/server/index.test.ts b/src/plugins/kibana_usage_collection/server/plugin.test.ts similarity index 59% rename from src/plugins/kibana_usage_collection/server/index.test.ts rename to src/plugins/kibana_usage_collection/server/plugin.test.ts index ee6df366b788f..cfae9d50d55c6 100644 --- a/src/plugins/kibana_usage_collection/server/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.test.ts @@ -14,7 +14,7 @@ import { import { CollectorOptions, createUsageCollectionSetupMock, -} from '../../usage_collection/server/usage_collection.mock'; +} from '../../usage_collection/server/mocks'; import { plugin } from './'; @@ -33,13 +33,63 @@ describe('kibana_usage_collection', () => { return createUsageCollectionSetupMock().makeStatsCollector(opts); }); - test('Runs the setup method without issues', () => { + test('Runs the setup method without issues', async () => { const coreSetup = coreMock.createSetup(); expect(pluginInstance.setup(coreSetup, { usageCollection })).toBe(undefined); - usageCollectors.forEach(({ isReady }) => { - expect(isReady()).toMatchSnapshot(); // Some should return false at this stage - }); + + await expect( + Promise.all( + usageCollectors.map(async (usageCollector) => { + const isReady = await usageCollector.isReady(); + const type = usageCollector.type; + return { type, isReady }; + }) + ) + ).resolves.toMatchInlineSnapshot(` + Array [ + Object { + "isReady": true, + "type": "ui_counters", + }, + Object { + "isReady": true, + "type": "usage_counters", + }, + Object { + "isReady": false, + "type": "kibana_stats", + }, + Object { + "isReady": true, + "type": "kibana", + }, + Object { + "isReady": false, + "type": "stack_management", + }, + Object { + "isReady": false, + "type": "ui_metric", + }, + Object { + "isReady": false, + "type": "application_usage", + }, + Object { + "isReady": true, + "type": "csp", + }, + Object { + "isReady": false, + "type": "core", + }, + Object { + "isReady": true, + "type": "localization", + }, + ] + `); }); test('Runs the start method without issues', () => { diff --git a/src/plugins/usage_collection/server/config.ts b/src/plugins/usage_collection/server/config.ts index a1b9a6165e1b5..87d86655ff917 100644 --- a/src/plugins/usage_collection/server/config.ts +++ b/src/plugins/usage_collection/server/config.ts @@ -14,7 +14,7 @@ export const configSchema = schema.object({ usageCounters: schema.object({ enabled: schema.boolean({ defaultValue: true }), retryCount: schema.number({ defaultValue: 1 }), - bufferDebounceMs: schema.number({ defaultValue: 30 * 1000 }), + bufferDebounceMs: schema.number({ defaultValue: 5 * 1000 }), }), uiCounters: schema.object({ enabled: schema.boolean({ defaultValue: true }), diff --git a/src/plugins/usage_collection/server/mocks.ts b/src/plugins/usage_collection/server/mocks.ts index e5ad102263626..fcaad6ac32558 100644 --- a/src/plugins/usage_collection/server/mocks.ts +++ b/src/plugins/usage_collection/server/mocks.ts @@ -6,20 +6,58 @@ * Side Public License, v 1. */ -import { loggingSystemMock } from '../../../core/server/mocks'; -import { UsageCollectionSetup } from './plugin'; -import { CollectorSet } from './collector'; -export { Collector, createCollectorFetchContextMock } from './usage_collection.mock'; - -const createSetupContract = () => { - return { - ...new CollectorSet({ - logger: loggingSystemMock.createLogger(), - maximumWaitTimeForAllCollectorsInS: 1, - }), - } as UsageCollectionSetup; +import { + elasticsearchServiceMock, + httpServerMock, + loggingSystemMock, + savedObjectsClientMock, +} from '../../../../src/core/server/mocks'; + +import { CollectorOptions, Collector, UsageCollector } from './collector'; +import { UsageCollectionSetup, CollectorFetchContext } from './index'; + +export type { CollectorOptions }; +export { Collector }; + +const logger = loggingSystemMock.createLogger(); + +export const createUsageCollectionSetupMock = () => { + const usageCollectionSetupMock: jest.Mocked = { + areAllCollectorsReady: jest.fn(), + bulkFetch: jest.fn(), + createUsageCounter: jest.fn(), + getUsageCounterByType: jest.fn(), + getCollectorByType: jest.fn(), + toApiFieldNames: jest.fn(), + toObject: jest.fn(), + makeStatsCollector: jest.fn().mockImplementation((cfg) => new Collector(logger, cfg)), + makeUsageCollector: jest.fn().mockImplementation((cfg) => new UsageCollector(logger, cfg)), + registerCollector: jest.fn(), + }; + + usageCollectionSetupMock.areAllCollectorsReady.mockResolvedValue(true); + return usageCollectionSetupMock; }; +export function createCollectorFetchContextMock(): jest.Mocked> { + const collectorFetchClientsMock: jest.Mocked> = { + esClient: elasticsearchServiceMock.createClusterClient().asInternalUser, + soClient: savedObjectsClientMock.create(), + }; + return collectorFetchClientsMock; +} + +export function createCollectorFetchContextWithKibanaMock(): jest.Mocked< + CollectorFetchContext +> { + const collectorFetchClientsMock: jest.Mocked> = { + esClient: elasticsearchServiceMock.createClusterClient().asInternalUser, + soClient: savedObjectsClientMock.create(), + kibanaRequest: httpServerMock.createKibanaRequest(), + }; + return collectorFetchClientsMock; +} + export const usageCollectionPluginMock = { - createSetupContract, + createSetupContract: createUsageCollectionSetupMock, }; diff --git a/src/plugins/usage_collection/server/report/store_report.test.ts b/src/plugins/usage_collection/server/report/store_report.test.ts index 5574220c7358d..08fdec4ae804f 100644 --- a/src/plugins/usage_collection/server/report/store_report.test.ts +++ b/src/plugins/usage_collection/server/report/store_report.test.ts @@ -13,15 +13,11 @@ import { storeReport } from './store_report'; import { ReportSchemaType } from './schema'; import { METRIC_TYPE } from '@kbn/analytics'; import { usageCountersServiceMock } from '../usage_counters/usage_counters_service.mock'; -import moment from 'moment'; describe('store_report', () => { const usageCountersServiceSetup = usageCountersServiceMock.createSetupContract(); const uiCountersUsageCounter = usageCountersServiceSetup.createUsageCounter('uiCounter'); - const momentTimestamp = moment(); - const date = momentTimestamp.format('DDMMYYYY'); - let repository: ReturnType; beforeEach(() => { @@ -99,7 +95,8 @@ describe('store_report', () => { ], ] `); - expect(uiCountersUsageCounter.incrementCounter.mock.calls).toMatchInlineSnapshot(` + expect((uiCountersUsageCounter.incrementCounter as jest.Mock).mock.calls) + .toMatchInlineSnapshot(` Array [ Array [ Object { diff --git a/src/plugins/usage_collection/server/routes/index.ts b/src/plugins/usage_collection/server/routes/index.ts index 3591ca2d59f9c..20949224c0f6d 100644 --- a/src/plugins/usage_collection/server/routes/index.ts +++ b/src/plugins/usage_collection/server/routes/index.ts @@ -16,7 +16,7 @@ import { Observable } from 'rxjs'; import { CollectorSet } from '../collector'; import { registerUiCountersRoute } from './ui_counters'; import { registerStatsRoute } from './stats'; -import { UsageCounter } from '../usage_counters'; +import type { UsageCounter } from '../usage_counters'; export function setupRoutes({ router, uiCountersUsageCounter, diff --git a/src/plugins/usage_collection/server/usage_collection.mock.ts b/src/plugins/usage_collection/server/usage_collection.mock.ts deleted file mode 100644 index 7e3f4273bbea8..0000000000000 --- a/src/plugins/usage_collection/server/usage_collection.mock.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { - elasticsearchServiceMock, - httpServerMock, - loggingSystemMock, - savedObjectsClientMock, -} from '../../../../src/core/server/mocks'; - -import { CollectorOptions, Collector, UsageCollector } from './collector'; -import { UsageCollectionSetup, CollectorFetchContext } from './index'; - -export type { CollectorOptions }; -export { Collector }; - -const logger = loggingSystemMock.createLogger(); - -export const createUsageCollectionSetupMock = () => { - const usageCollectionSetupMock: jest.Mocked = { - areAllCollectorsReady: jest.fn(), - bulkFetch: jest.fn(), - bulkFetchUsage: jest.fn(), - getCollectorByType: jest.fn(), - toApiFieldNames: jest.fn(), - toObject: jest.fn(), - makeStatsCollector: jest.fn().mockImplementation((cfg) => new Collector(logger, cfg)), - makeUsageCollector: jest.fn().mockImplementation((cfg) => new UsageCollector(logger, cfg)), - registerCollector: jest.fn(), - }; - - usageCollectionSetupMock.areAllCollectorsReady.mockResolvedValue(true); - return usageCollectionSetupMock; -}; - -export function createCollectorFetchContextMock(): jest.Mocked> { - const collectorFetchClientsMock: jest.Mocked> = { - esClient: elasticsearchServiceMock.createClusterClient().asInternalUser, - soClient: savedObjectsClientMock.create(), - }; - return collectorFetchClientsMock; -} - -export function createCollectorFetchContextWithKibanaMock(): jest.Mocked< - CollectorFetchContext -> { - const collectorFetchClientsMock: jest.Mocked> = { - esClient: elasticsearchServiceMock.createClusterClient().asInternalUser, - soClient: savedObjectsClientMock.create(), - kibanaRequest: httpServerMock.createKibanaRequest(), - }; - return collectorFetchClientsMock; -} diff --git a/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts index 5992674ec0363..4d9c3e56d5d1d 100644 --- a/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts +++ b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts @@ -38,6 +38,12 @@ describe('counterKey', () => { describe('storeCounter', () => { const internalRepository = savedObjectsRepositoryMock.create(); + const mockNow = 1617954426939; + + beforeEach(() => { + jest.spyOn(moment, 'now').mockReturnValue(mockNow); + }); + afterAll(() => { jest.resetAllMocks(); }); diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts index d119a13b292d0..8ebfc66070581 100644 --- a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts @@ -5,6 +5,8 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ + +/* eslint-disable dot-notation */ import { UsageCountersService } from './usage_counters_service'; import { loggingSystemMock, coreMock } from '../../../../core/server/mocks'; import * as rxOp from 'rxjs/operators'; @@ -40,15 +42,15 @@ describe('UsageCountersService', () => { usageCounter.incrementCounter({ counterName: 'counterA' }); usageCounter.incrementCounter({ counterName: 'counterA' }); - const dataInSourcePromise = usageCountersService.source$.pipe(rxOp.toArray()).toPromise(); - usageCountersService.flushCache$.next(); - usageCountersService.source$.complete(); + const dataInSourcePromise = usageCountersService['source$'].pipe(rxOp.toArray()).toPromise(); + usageCountersService['flushCache$'].next(); + usageCountersService['source$'].complete(); await expect(dataInSourcePromise).resolves.toHaveLength(2); }); it('registers savedObject type during setup', () => { const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDebounceMs }); - const { createUsageCounter } = usageCountersService.setup(coreSetup); + usageCountersService.setup(coreSetup); expect(coreSetup.savedObjects.registerType).toBeCalledTimes(1); }); @@ -67,9 +69,9 @@ describe('UsageCountersService', () => { usageCounter.incrementCounter({ counterName: 'counterA' }); usageCounter.incrementCounter({ counterName: 'counterA' }); - const dataInSourcePromise = usageCountersService.source$.pipe(rxOp.toArray()).toPromise(); + const dataInSourcePromise = usageCountersService['source$'].pipe(rxOp.toArray()).toPromise(); usageCountersService.start(coreStart); - usageCountersService.source$.complete(); + usageCountersService['source$'].complete(); await expect(dataInSourcePromise).resolves.toMatchInlineSnapshot(` Array [ diff --git a/src/plugins/vis_type_table/server/usage_collector/register_usage_collector.test.ts b/src/plugins/vis_type_table/server/usage_collector/register_usage_collector.test.ts index b87e6d54733af..e045788897b61 100644 --- a/src/plugins/vis_type_table/server/usage_collector/register_usage_collector.test.ts +++ b/src/plugins/vis_type_table/server/usage_collector/register_usage_collector.test.ts @@ -10,8 +10,10 @@ jest.mock('./get_stats', () => ({ getStats: jest.fn().mockResolvedValue({ somestat: 1 }), })); -import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock'; -import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; +import { + createUsageCollectionSetupMock, + createCollectorFetchContextMock, +} from 'src/plugins/usage_collection/server/mocks'; import { registerVisTypeTableUsageCollector } from './register_usage_collector'; import { getStats } from './get_stats'; diff --git a/src/plugins/vis_type_timeseries/server/usage_collector/register_timeseries_collector.test.ts b/src/plugins/vis_type_timeseries/server/usage_collector/register_timeseries_collector.test.ts index 2612a3882af2d..726ad972ab8d1 100644 --- a/src/plugins/vis_type_timeseries/server/usage_collector/register_timeseries_collector.test.ts +++ b/src/plugins/vis_type_timeseries/server/usage_collector/register_timeseries_collector.test.ts @@ -8,7 +8,7 @@ import { of } from 'rxjs'; import { mockStats, mockGetStats } from './get_usage_collector.mock'; -import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock'; +import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/mocks'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { registerTimeseriesUsageCollector } from './register_timeseries_collector'; import { ConfigObservable } from '../types'; diff --git a/src/plugins/vis_type_vega/server/usage_collector/register_vega_collector.test.ts b/src/plugins/vis_type_vega/server/usage_collector/register_vega_collector.test.ts index 9db1b7657f444..7933da3e675f6 100644 --- a/src/plugins/vis_type_vega/server/usage_collector/register_vega_collector.test.ts +++ b/src/plugins/vis_type_vega/server/usage_collector/register_vega_collector.test.ts @@ -8,7 +8,7 @@ import { of } from 'rxjs'; import { mockStats, mockGetStats } from './get_usage_collector.mock'; -import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock'; +import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/mocks'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { HomeServerPluginSetup } from '../../../home/server'; import { registerVegaUsageCollector } from './register_vega_collector'; diff --git a/src/plugins/visualizations/server/usage_collector/register_visualizations_collector.test.ts b/src/plugins/visualizations/server/usage_collector/register_visualizations_collector.test.ts index 743ec29fe9af7..a3617631f734b 100644 --- a/src/plugins/visualizations/server/usage_collector/register_visualizations_collector.test.ts +++ b/src/plugins/visualizations/server/usage_collector/register_visualizations_collector.test.ts @@ -8,7 +8,7 @@ import { of } from 'rxjs'; import { mockStats, mockGetStats } from './get_usage_collector.mock'; -import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/usage_collection.mock'; +import { createUsageCollectionSetupMock } from 'src/plugins/usage_collection/server/mocks'; import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; import { registerVisualizationsCollector } from './register_visualizations_collector'; From a76bbd7c19e43a4dbff6f89cfb1ed0935afb47bc Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Sun, 11 Apr 2021 11:35:19 +0300 Subject: [PATCH 03/11] typecheck --- .../server/collectors/ui_counters/rollups/rollups.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts index 4a6b55730a5df..6c58884a1a259 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts @@ -81,7 +81,9 @@ describe('rollUiCounterIndices', () => { }); it('returns undefined if no savedObjectsClient initialised yet', async () => { - await expect(rollUiCounterIndices(logger, undefined)).resolves.toBe(undefined); + await expect( + rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, undefined) + ).resolves.toBe(undefined); expect(logger.warn).toHaveBeenCalledTimes(0); }); From 7ccb90b306fcaf787acf9d77aafb58329046eab5 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Sun, 11 Apr 2021 19:10:39 +0300 Subject: [PATCH 04/11] fix tests --- ...rver.indexpatternsserviceprovider.start.md | 4 +-- src/plugins/data/server/server.api.md | 1 + src/plugins/usage_collection/server/mocks.ts | 25 +++++++++++-------- .../usage_collection/usage_counters.ts | 6 ++--- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md index 88079bb2fa3cb..118b0104fbee6 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md @@ -8,7 +8,7 @@ ```typescript start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps): { - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; }; ``` @@ -22,6 +22,6 @@ start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps): Returns: `{ - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: ElasticsearchClient) => Promise; }` diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 0ea3af60e9b5d..622356c4441ac 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -56,6 +56,7 @@ import { PublicMethodsOf } from '@kbn/utility-types'; import { RecursiveReadonly } from '@kbn/utility-types'; import { RequestAdapter } from 'src/plugins/inspector/common'; import { RequestHandlerContext } from 'src/core/server'; +import * as Rx from 'rxjs'; import { SavedObject } from 'kibana/server'; import { SavedObject as SavedObject_2 } from 'src/core/server'; import { SavedObjectsClientContract } from 'src/core/server'; diff --git a/src/plugins/usage_collection/server/mocks.ts b/src/plugins/usage_collection/server/mocks.ts index fcaad6ac32558..b84fa0f0aab70 100644 --- a/src/plugins/usage_collection/server/mocks.ts +++ b/src/plugins/usage_collection/server/mocks.ts @@ -13,26 +13,29 @@ import { savedObjectsClientMock, } from '../../../../src/core/server/mocks'; -import { CollectorOptions, Collector, UsageCollector } from './collector'; +import { CollectorOptions, Collector, CollectorSet } from './collector'; import { UsageCollectionSetup, CollectorFetchContext } from './index'; export type { CollectorOptions }; export { Collector }; -const logger = loggingSystemMock.createLogger(); - export const createUsageCollectionSetupMock = () => { + const collectorSet = new CollectorSet({ + logger: loggingSystemMock.createLogger(), + maximumWaitTimeForAllCollectorsInS: 1, + }); + const usageCollectionSetupMock: jest.Mocked = { - areAllCollectorsReady: jest.fn(), - bulkFetch: jest.fn(), createUsageCounter: jest.fn(), getUsageCounterByType: jest.fn(), - getCollectorByType: jest.fn(), - toApiFieldNames: jest.fn(), - toObject: jest.fn(), - makeStatsCollector: jest.fn().mockImplementation((cfg) => new Collector(logger, cfg)), - makeUsageCollector: jest.fn().mockImplementation((cfg) => new UsageCollector(logger, cfg)), - registerCollector: jest.fn(), + areAllCollectorsReady: jest.fn().mockImplementation(collectorSet.areAllCollectorsReady), + bulkFetch: jest.fn().mockImplementation(collectorSet.bulkFetch), + getCollectorByType: jest.fn().mockImplementation(collectorSet.getCollectorByType), + toApiFieldNames: jest.fn().mockImplementation(collectorSet.toApiFieldNames), + toObject: jest.fn().mockImplementation(collectorSet.toObject), + makeStatsCollector: jest.fn().mockImplementation(collectorSet.makeStatsCollector), + makeUsageCollector: jest.fn().mockImplementation(collectorSet.makeUsageCollector), + registerCollector: jest.fn().mockImplementation(collectorSet.registerCollector), }; usageCollectionSetupMock.areAllCollectorsReady.mockResolvedValue(true); diff --git a/test/plugin_functional/test_suites/usage_collection/usage_counters.ts b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts index 444c0edbac5e2..a5ac4324369be 100644 --- a/test/plugin_functional/test_suites/usage_collection/usage_counters.ts +++ b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts @@ -18,6 +18,9 @@ export default function ({ getService, getPageObjects }: PluginFunctionalProvide const supertest = getService('supertest'); async function getSavedObjectCounters() { + // wait until ES indexes the counter SavedObject; + await new Promise((res) => setTimeout(res, 7 * 1000)); + return await supertest .get('/api/saved_objects/_find?type=usage-counters') .set('kbn-xsrf', 'true') @@ -63,9 +66,6 @@ export default function ({ getService, getPageObjects }: PluginFunctionalProvide .set('kbn-xsrf', 'true') .expect(200); - // wait until ES indexes the counter SavedObject; - await new Promise((res) => setTimeout(res, 7 * 1000)); - const { routeAccessed } = await getSavedObjectCounters(); expect(routeAccessed).to.be(1); }); From df928a96ada69b21340f7adae8a7e87aab6b3872 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Sun, 11 Apr 2021 22:06:28 +0300 Subject: [PATCH 05/11] fix functional --- test/plugin_functional/config.ts | 2 +- .../plugins/usage_collection/server/plugin.ts | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/plugin_functional/config.ts b/test/plugin_functional/config.ts index c51de3b674647..d8bab12f7c037 100644 --- a/test/plugin_functional/config.ts +++ b/test/plugin_functional/config.ts @@ -21,6 +21,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { return { testFiles: [ + require.resolve('./test_suites/usage_collection'), require.resolve('./test_suites/core'), require.resolve('./test_suites/custom_visualizations'), require.resolve('./test_suites/panel_actions'), @@ -31,7 +32,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('./test_suites/data_plugin'), require.resolve('./test_suites/saved_objects_management'), require.resolve('./test_suites/saved_objects_hidden_type'), - require.resolve('./test_suites/usage_collection'), ], services: { ...functionalConfig.get('services'), diff --git a/test/plugin_functional/plugins/usage_collection/server/plugin.ts b/test/plugin_functional/plugins/usage_collection/server/plugin.ts index 0ce60bd5c21e3..32e21950d4722 100644 --- a/test/plugin_functional/plugins/usage_collection/server/plugin.ts +++ b/test/plugin_functional/plugins/usage_collection/server/plugin.ts @@ -25,7 +25,7 @@ export class UsageCollectionTestPlugin implements Plugin { throw new Error('Optional plugin `usageCollection` is expected to be enabled for testing.'); } - const usageCounter = usageCollection?.createUsageCounter('usageCollectionTestPlugin'); + const usageCounter = usageCollection.createUsageCounter('usageCollectionTestPlugin'); registerRoutes(core.http, usageCounter); usageCounter.incrementCounter({ @@ -37,7 +37,10 @@ export class UsageCollectionTestPlugin implements Plugin { } public start() { - this.usageCounter?.incrementCounter({ counterName: 'duringStart' }); + if (!this.usageCounter) { + throw new Error('Optional plugin `usageCollection` is expected to be enabled for testing.'); + } + this.usageCounter.incrementCounter({ counterName: 'duringStart' }); } public stop() {} From 204134e736cad5338fc1b908ae0756ff5e4b9085 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Mon, 12 Apr 2021 11:57:25 +0300 Subject: [PATCH 06/11] add documentation --- src/plugins/usage_collection/README.mdx | 105 +++++++++++++++++++++++- 1 file changed, 102 insertions(+), 3 deletions(-) diff --git a/src/plugins/usage_collection/README.mdx b/src/plugins/usage_collection/README.mdx index 04e1e0fbb5006..2cd41e5d9e0aa 100644 --- a/src/plugins/usage_collection/README.mdx +++ b/src/plugins/usage_collection/README.mdx @@ -20,6 +20,7 @@ The way to report the usage of any feature depends on whether the actions to tra In any case, to use any of these APIs, the plugin must optionally require the plugin `usageCollection`: + ```json // plugin/kibana.json { @@ -112,6 +113,104 @@ Not an API as such. However, Data Telemetry collects the usage of known patterns This collector does not report the name of the indices nor any content. It only provides stats about usage of known shippers/ingest tools. +#### Usage Counters + +Usage counters allows plugins to report user triggered events from the server. This api has feature parity with UI Counters on the `public` plugin side of usage_collection. + +Usage counters provide instrumentation on the server to count triggered events such as "api called", "threshold reached", and miscellaneous events count. + +It is useful for gathering _semi-aggregated_ events with a per day granularity. +This allows tracking trends in usage and provides enough granularity for this type of telemetry to provide insights such as +- "How many times this threshold has been reached" +- "What is the trend in usage of this api" +- "How frequent are users hitting this error per day" +- "What is the success rate of this operation" +- "Which option is being selected the most/least" + +##### How to use it + +To create a usage counter for your plugin, use the API `usageCollection.createUsageCounter` as follows: + +```ts +// server/plugin.ts +import { Plugin, CoreStart } from '../../../core/public'; +import { UsageCollectionSetup, UsageCounter } from '../../../plugins/usage_collection/server'; + +export class MyPlugin implements Plugin { + private usageCounter?: UsageCounter; + public setup( + core: CoreStart, + { usageCollection }: { usageCollection?: UsageCollectionSetup } + ) { + + /** + * Create a usage counter for this plugin. Domain ID must be unique. + * It is advised to use the plugin name as the domain ID for most cases. + */ + this.usageCounter = usageCollection?.createUsageCounter(''); + try { + doSomeOperation(); + this.usageCounter?.incrementCounter({ + counterName: 'doSomeOperation_success', + incrementBy: 1, + }); + } catch (err) { + this.usageCounter?.incrementCounter({ + counterName: 'doSomeOperation_error', + counterType: 'error', + incrementBy: 1, + }); + logger.error(err); + } + } +} +``` + +Pass the created `usageCounter` around in your service to instrument usage. + +That's all you need to do! The Usage counters service will handle piping these counters all the way to the telemetry service. + +##### Telemetry reported usage + +Usage counters are reported inside the telemetry usage payload under `stack_stats.kibana.plugins.usage_counters`. + +```ts +{ + usage_counters: { + dailyEvents: [ + { + domainId: '', + counterName: 'doSomeOperation_success', + counterType: 'count', + lastUpdatedAt: '2021-11-20T11:43:00.961Z', + fromTimestamp: '2021-11-20T00:00:00Z', + total: 3, + }, + { + domainId: '', + counterName: 'doSomeOperation_success', + counterType: 'count', + lastUpdatedAt: '2021-11-21T10:30:00.961Z', + fromTimestamp: '2021-11-21T00:00:00Z', + total: 5, + }, + { + domainId: '', + counterName: 'doSomeOperation_error', + counterType: 'error', + lastUpdatedAt: '2021-11-20T11:43:00.961Z', + fromTimestamp: '2021-11-20T00:00:00Z', + total: 1, + }, + ], + }, +} +``` + +##### Disallowed characters + +The colon character (`:`) should not be used in the `counterType`. Colons play a special role for `counterType` in how metrics are stored as saved objects. + #### Custom collector In many cases, plugins need to report the custom usage of a feature. In this cases, the plugins must complete the following 2 steps in the `setup` lifecycle step: @@ -202,7 +301,7 @@ Some background: - `isReady` (added in v7.2.0 and v6.8.4) is a way for a usage collector to announce that some async process must finish first before it can return data in the `fetch` method (e.g. a client needs to ne initialized, or the task manager needs to run a task first). If any collector reports that it is not ready when we call its `fetch` method, we reset a flag to try again and, after a set amount of time, collect data from those collectors that are ready and skip any that are not. This means that if a collector returns `true` for `isReady` and it actually isn't ready to return data, there won't be telemetry data from that collector in that telemetry report (usually once per day). You should consider what it means if your collector doesn't return data in the first few documents when Kibana starts or, if we should wait for any other reason (e.g. the task manager needs to run your task first). If you need to tell telemetry collection to wait, you should implement this function with custom logic. If your `fetch` method can run without the need of any previous dependencies, then you can return true for `isReady` as shown in the example below. -- The `fetch` method needs to support multiple contexts in which it is called. For example, when a user requests the example of what we collect in the **Kibana>Advanced Settings>Usage data** section, the clients provided in the context of the function (`CollectorFetchContext`) are scoped to that user's privileges. The reason is to avoid exposing via telemetry any data that user should not have access to (i.e.: if the user does not have access to certain indices, they shouldn't be allowed to see the number of documents that exists in it). In this case, the `fetch` method receives the clients `esClient` and `soClient` scoped to the user who performed the HTTP API request. Alternatively, when requesting the usage data to be reported to the Remote Telemetry Service, the clients are scoped to the internal Kibana user (`kibana_system`). Please, mind it might have lower-level access than the default super-admin `elastic` test user. +- The `fetch` method needs to support multiple contexts in which it is called. For example, when a user requests the example of what we collect in the **Kibana>Advanced Settings>Usage data** section, the clients provided in the context of the function (`CollectorFetchContext`) are scoped to that user's privileges. The reason is to avoid exposing via telemetry any data that user should not have access to (i.e.: if the user does not have access to certain indices, they shouldn't be allowed to see the number of documents that exists in it). In this case, the `fetch` method receives the clients `esClient` and `soClient` scoped to the user who performed the HTTP API request. Alternatively, when requesting the usage data to be reported to the Remote Telemetry Service, the clients are scoped to the internal Kibana user (`kibana_system`). Please, mind it might have lower-level access than the default super-admin `elastic` test user. In some scenarios, your collector might need to maintain its own client. An example of that is the `monitoring` plugin, that maintains a connection to the Remote Monitoring Cluster to push its monitoring data. If that's the case, your plugin can opt-in to receive the additional `kibanaRequest` parameter by adding `extendFetchContext.kibanaRequest: true` to the collector's config: it will be appended to the context of the `fetch` method only if the request needs to be scoped to a user other than Kibana Internal, so beware that your collector will need to work for both scenarios (especially for the scenario when `kibanaRequest` is missing). Note: there will be many cases where you won't need to use the `esClient` or `soClient` function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS. @@ -403,9 +502,9 @@ There are a few ways you can test that your usage collector is working properly. ## FAQ -1. **How should I design my data model?** +1. **How should I design my data model?** Keep it simple, and keep it to a model that Kibana will be able to understand. Bear in mind the number of keys you are reporting as it may result in fields mapping explosion. Flat arrays, such as arrays of strings are fine. -2. **If I accumulate an event counter in server memory, which my fetch method returns, won't it reset when the Kibana server restarts?** +2. **If I accumulate an event counter in server memory, which my fetch method returns, won't it reset when the Kibana server restarts?** Yes, but that is not a major concern. A visualization on such info might be a date histogram that gets events-per-second or something, which would be impacted by server restarts, so we'll have to offset the beginning of the time range when we detect that the latest metric is smaller than the earliest metric. That would be a pretty custom visualization, but perhaps future Kibana enhancements will be able to support that. # Routes registered by this plugin From 278114078f85661b806d9bd29986e81c5d0c9849 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 13 Apr 2021 19:56:59 +0300 Subject: [PATCH 07/11] duration config --- .../usage_counter_saved_objects.ts | 58 +++++++++++++++--- .../register_ui_counters_collector.ts | 39 +++++------- .../ui_counters/rollups/register_rollups.ts | 3 +- .../usage_counter_saved_objects.ts | 60 ++++++++++++++++--- .../register_usage_counters_collector.ts | 9 +-- .../usage_counters/rollups/rollups.ts | 2 +- .../kibana_usage_collection/server/plugin.ts | 11 +++- src/plugins/usage_collection/README.mdx | 24 ++++---- src/plugins/usage_collection/server/config.ts | 2 +- src/plugins/usage_collection/server/index.ts | 2 +- src/plugins/usage_collection/server/plugin.ts | 2 +- .../server/usage_counters/index.ts | 7 +-- .../usage_counters/saved_objects.test.ts | 20 +++---- .../server/usage_counters/saved_objects.ts | 33 +++++----- .../server/usage_counters/usage_counter.ts | 6 +- .../usage_counters_service.test.ts | 60 ++++++++++++------- .../usage_counters/usage_counters_service.ts | 17 ++++-- test/api_integration/config.js | 2 +- .../saved_objects/ui_counters/data.json | 10 +++- .../saved_objects/ui_counters/mappings.json | 6 +- .../saved_objects/usage_counters/data.json | 25 ++++++-- .../usage_counters/mappings.json | 6 +- test/plugin_functional/config.ts | 2 +- .../plugins/usage_collection/kibana.json | 2 +- .../plugins/usage_collection/server/config.ts | 18 ------ .../plugins/usage_collection/server/index.ts | 2 - .../plugins/usage_collection/server/plugin.ts | 8 +-- .../plugins/usage_collection/server/routes.ts | 4 +- .../usage_collection/usage_counters.ts | 10 +--- 29 files changed, 268 insertions(+), 182 deletions(-) delete mode 100644 test/plugin_functional/plugins/usage_collection/server/config.ts diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts index b828bb68153c4..e7baca772c549 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts @@ -12,7 +12,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'uiCounter:09042021:count:myApp:my_event', - attributes: { count: 1 }, + attributes: { + count: 1, + counterName: 'myApp:my_event', + counterType: 'count', + timestamp: '2021-04-09T08:17:57.693Z', + domainId: 'uiCounter', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:17:57.693Z', @@ -20,7 +26,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'uiCounter:09042021:loaded:Kibana_home:home_tutorial_directory', - attributes: { count: 60 }, + attributes: { + count: 60, + counterName: 'Kibana_home:home_tutorial_directory', + counterType: 'loaded', + timestamp: '2020-10-23T11:27:57.067Z', + domainId: 'uiCounter', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2020-10-23T11:27:57.067Z', @@ -28,7 +40,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'uiCounter:09042021:count:myApp:my_event_4457914848544', - attributes: { count: 0 }, + attributes: { + count: 0, + counterName: 'myApp:my_event_4457914848544', + counterType: 'count', + timestamp: '2021-04-09T08:18:03.030Z', + domainId: 'uiCounter', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', @@ -36,8 +54,14 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'uiCounter:09042021:count:myApp:my_event_malformed', - // @ts-expect-error - attributes: { count: 'malformed' }, + attributes: { + // @ts-expect-error + count: 'malformed', + counterName: 'myApp:my_event_malformed', + counterType: 'count', + timestamp: '2021-04-09T08:18:03.030Z', + domainId: 'uiCounter', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', @@ -45,7 +69,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'anotherDomainId:09042021:count:some_event_name', - attributes: { count: 4 }, + attributes: { + count: 4, + counterName: 'some_event_name', + counterType: 'count', + timestamp: '2021-04-09T08:18:03.030Z', + domainId: 'anotherDomainId', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', @@ -53,7 +83,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'uiCounter:09042021:count:myApp:my_event_4457914848544_2', - attributes: { count: 8 }, + attributes: { + count: 8, + counterName: 'myApp:my_event_4457914848544_2', + counterType: 'count', + timestamp: '2021-04-09T08:18:03.031Z', + domainId: 'uiCounter', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.031Z', @@ -61,7 +97,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'uiCounter:09042021:count:myApp:only_reported_in_usage_counters', - attributes: { count: 1 }, + attributes: { + count: 1, + counterName: 'myApp:only_reported_in_usage_counters', + counterType: 'count', + timestamp: '2021-04-09T08:18:03.031Z', + domainId: 'uiCounter', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.031Z', diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts index dbe35a35ebe94..7324e49da6ec0 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts @@ -7,7 +7,6 @@ */ import moment from 'moment'; -import { chain } from 'lodash'; import { UICounterSavedObject, @@ -21,7 +20,6 @@ import { USAGE_COUNTERS_SAVED_OBJECT_TYPE, UsageCountersSavedObject, UsageCountersSavedObjectAttributes, - deserializeCounterKey, serializeCounterKey, } from '../../../../usage_collection/server'; @@ -73,11 +71,9 @@ export function transformRawUsageCounterObject( rawUsageCounter: UsageCountersSavedObject ): UiCounterEvent | undefined { const { - id, - attributes: { count }, + attributes: { count, counterName, counterType, domainId }, updated_at: lastUpdatedAt, } = rawUsageCounter; - const { counterName, counterType, domainId } = deserializeCounterKey(id); if (domainId !== 'uiCounter' || typeof count !== 'number' || count < 1) { return; @@ -101,7 +97,8 @@ export async function fetchUiCounters({ soClient }: CollectorFetchContext) { saved_objects: rawUsageCounters, } = await soClient.find({ type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, - fields: ['count'], + fields: ['count', 'counterName', 'counterType', 'domainId'], + filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`, perPage: 10000, }); @@ -143,24 +140,18 @@ export async function fetchUiCounters({ soClient }: CollectorFetchContext) { return acc; }, {} as Record); - return { - dailyEvents: chain(dailyEventsFromUsageCounters) - .mergeWith( - dailyEventsFromUiCounters, - (value: UiCounterEvent | undefined, srcValue: UiCounterEvent): UiCounterEvent => { - if (!value) { - return srcValue; - } - - return { - ...srcValue, - total: srcValue.total + value.total, - }; - } - ) - .values() - .value(), - }; + const mergedDailyCounters = Object.entries(dailyEventsFromUiCounters).reduce( + (acc, [key, value]) => { + if (acc[key]) { + value.total = acc[key].total + value.total; + } + acc[key] = value; + return acc; + }, + dailyEventsFromUsageCounters + ); + + return { dailyEvents: Object.values(mergedDailyCounters) }; } export function registerUiCountersUsageCollector(usageCollection: UsageCollectionSetup) { diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts index 85d31dd30576a..55da239d8ef2a 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts @@ -12,10 +12,9 @@ import { Logger, ISavedObjectsRepository } from 'kibana/server'; import { ROLL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants'; import { rollUiCounterIndices } from './rollups'; -export const stopRollingUiCounterIndicies$ = new Subject(); - export function registerUiCountersRollups( logger: Logger, + stopRollingUiCounterIndicies$: Subject, getSavedObjectsClient: () => ISavedObjectsRepository | undefined ) { timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL) diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts index 99f23bb08de23..b739744e24224 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts @@ -11,8 +11,14 @@ import type { UsageCountersSavedObject } from '../../../../../usage_collection/s export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', - id: 'uiCounter:09042021:count:myApp:my_event_malformed', - attributes: { count: 13 }, + id: 'uiCounter:09042021:count:myApp:my_event', + attributes: { + count: 13, + counterName: 'my_event', + counterType: 'count', + domainId: 'uiCounter', + timestamp: '2021-04-09T08:18:03.030Z', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', @@ -20,7 +26,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'anotherDomainId:09042021:count:some_event_name', - attributes: { count: 4 }, + attributes: { + count: 4, + counterName: 'some_event_name', + counterType: 'count', + domainId: 'anotherDomainId', + timestamp: '2021-04-09T08:18:03.030Z', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-09T08:18:03.030Z', @@ -28,7 +40,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'anotherDomainId:09042021:count:some_event_name', - attributes: { count: 4 }, + attributes: { + count: 4, + counterName: 'some_event_name', + counterType: 'count', + domainId: 'anotherDomainId', + timestamp: '2021-04-11T08:18:03.030Z', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-11T08:18:03.030Z', @@ -36,7 +54,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'anotherDomainId2:09042021:count:some_event_name', - attributes: { count: 1 }, + attributes: { + count: 1, + counterName: 'some_event_name', + counterType: 'count', + domainId: 'anotherDomainId2', + timestamp: '2021-04-20T08:18:03.030Z', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-20T08:18:03.030Z', @@ -44,8 +68,14 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'anotherDomainId2:09042021:count:malformed_event', - // @ts-expect-error - attributes: { count: 'malformed' }, + attributes: { + // @ts-expect-error + count: 'malformed', + counterName: 'malformed_event', + counterType: 'count', + domainId: 'anotherDomainId2', + timestamp: '2021-04-20T08:18:03.030Z', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-20T08:18:03.030Z', @@ -53,7 +83,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'anotherDomainId2:09042021:custom_type:some_event_name', - attributes: { count: 3 }, + attributes: { + count: 3, + counterName: 'some_event_name', + counterType: 'custom_type', + domainId: 'anotherDomainId2', + timestamp: '2021-04-20T08:18:03.030Z', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-20T08:18:03.030Z', @@ -61,7 +97,13 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ { type: 'usage-counters', id: 'anotherDomainId3:09042021:custom_type:zero_count', - attributes: { count: 0 }, + attributes: { + count: 0, + counterName: 'zero_count', + counterType: 'custom_type', + domainId: 'anotherDomainId3', + timestamp: '2021-04-20T08:18:03.030Z', + }, references: [], coreMigrationVersion: '8.0.0', updated_at: '2021-04-20T08:18:03.030Z', diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts index b6f5fc7f842eb..9c6db00fb3597 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts @@ -13,7 +13,6 @@ import { USAGE_COUNTERS_SAVED_OBJECT_TYPE, UsageCountersSavedObject, UsageCountersSavedObjectAttributes, - deserializeCounterKey, } from '../../../../usage_collection/server'; interface UsageCounterEvent { @@ -33,11 +32,9 @@ export function transformRawCounter( rawUsageCounter: UsageCountersSavedObject ): UsageCounterEvent | undefined { const { - id, - attributes: { count }, + attributes: { count, counterName, counterType, domainId }, updated_at: lastUpdatedAt, } = rawUsageCounter; - const { domainId, counterType, counterName } = deserializeCounterKey(id); const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format(); if (domainId === 'uiCounter' || typeof count !== 'number' || count < 1) { @@ -93,7 +90,8 @@ export function registerUsageCountersUsageCollector(usageCollection: UsageCollec saved_objects: rawUsageCounters, } = await soClient.find({ type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, - fields: ['count'], + fields: ['count', 'counterName', 'counterType', 'domainId'], + filter: `NOT ${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`, perPage: 10000, }); @@ -104,7 +102,6 @@ export function registerUsageCountersUsageCollector(usageCollection: UsageCollec if (event) { acc.push(event); } - return acc; } catch (_) { // swallow error; allows sending successfully transformed objects. } diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts index 74b555f1c296b..c07ea37536f2d 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { ISavedObjectsRepository, Logger } from 'kibana/server'; +import type { ISavedObjectsRepository, Logger } from 'kibana/server'; import moment from 'moment'; import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index d91cefb6c8083..4ab3f5d88dffe 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -52,11 +52,13 @@ export class KibanaUsageCollectionPlugin implements Plugin { private uiSettingsClient?: IUiSettingsClient; private metric$: Subject; private coreUsageData?: CoreUsageDataStart; + private stopRollingUiCounterIndicies$: Subject; constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); this.legacyConfig$ = initializerContext.config.legacy.globalConfig$; this.metric$ = new Subject(); + this.stopRollingUiCounterIndicies$ = new Subject(); } public setup(coreSetup: CoreSetup, { usageCollection }: KibanaUsageCollectionPluginsDepsSetup) { @@ -66,6 +68,7 @@ export class KibanaUsageCollectionPlugin implements Plugin { usageCollection, coreSetup, this.metric$, + this.stopRollingUiCounterIndicies$, coreSetup.savedObjects.registerType.bind(coreSetup.savedObjects) ); } @@ -81,12 +84,14 @@ export class KibanaUsageCollectionPlugin implements Plugin { public stop() { this.metric$.complete(); + this.stopRollingUiCounterIndicies$.complete(); } private registerUsageCollectors( usageCollection: UsageCollectionSetup, coreSetup: CoreSetup, metric$: Subject, + stopRollingUiCounterIndicies$: Subject, registerType: SavedObjectsRegisterType ) { const getSavedObjectsClient = () => this.savedObjectsClient; @@ -94,7 +99,11 @@ export class KibanaUsageCollectionPlugin implements Plugin { const getCoreUsageDataService = () => this.coreUsageData!; registerUiCounterSavedObjectType(coreSetup.savedObjects); - registerUiCountersRollups(this.logger.get('ui-counters'), getSavedObjectsClient); + registerUiCountersRollups( + this.logger.get('ui-counters'), + stopRollingUiCounterIndicies$, + getSavedObjectsClient + ); registerUiCountersUsageCollector(usageCollection); registerUsageCountersRollups(this.logger.get('usage-counters-rollup'), getSavedObjectsClient); diff --git a/src/plugins/usage_collection/README.mdx b/src/plugins/usage_collection/README.mdx index 2cd41e5d9e0aa..6d53286a84dc1 100644 --- a/src/plugins/usage_collection/README.mdx +++ b/src/plugins/usage_collection/README.mdx @@ -121,11 +121,11 @@ Usage counters provide instrumentation on the server to count triggered events s It is useful for gathering _semi-aggregated_ events with a per day granularity. This allows tracking trends in usage and provides enough granularity for this type of telemetry to provide insights such as -- "How many times this threshold has been reached" -- "What is the trend in usage of this api" -- "How frequent are users hitting this error per day" -- "What is the success rate of this operation" -- "Which option is being selected the most/least" +- "How many times this threshold has been reached?" +- "What is the trend in usage of this api?" +- "How frequent are users hitting this error per day?" +- "What is the success rate of this operation?" +- "Which option is being selected the most/least?" ##### How to use it @@ -133,8 +133,8 @@ To create a usage counter for your plugin, use the API `usageCollection.createUs ```ts // server/plugin.ts -import { Plugin, CoreStart } from '../../../core/public'; -import { UsageCollectionSetup, UsageCounter } from '../../../plugins/usage_collection/server'; +import type { Plugin, CoreStart } from '../../../core/public'; +import type { UsageCollectionSetup, UsageCounter } from '../../../plugins/usage_collection/server'; export class MyPlugin implements Plugin { private usageCounter?: UsageCounter; @@ -207,10 +207,6 @@ Usage counters are reported inside the telemetry usage payload under `stack_stat } ``` -##### Disallowed characters - -The colon character (`:`) should not be used in the `counterType`. Colons play a special role for `counterType` in how metrics are stored as saved objects. - #### Custom collector In many cases, plugins need to report the custom usage of a feature. In this cases, the plugins must complete the following 2 steps in the `setup` lifecycle step: @@ -301,7 +297,7 @@ Some background: - `isReady` (added in v7.2.0 and v6.8.4) is a way for a usage collector to announce that some async process must finish first before it can return data in the `fetch` method (e.g. a client needs to ne initialized, or the task manager needs to run a task first). If any collector reports that it is not ready when we call its `fetch` method, we reset a flag to try again and, after a set amount of time, collect data from those collectors that are ready and skip any that are not. This means that if a collector returns `true` for `isReady` and it actually isn't ready to return data, there won't be telemetry data from that collector in that telemetry report (usually once per day). You should consider what it means if your collector doesn't return data in the first few documents when Kibana starts or, if we should wait for any other reason (e.g. the task manager needs to run your task first). If you need to tell telemetry collection to wait, you should implement this function with custom logic. If your `fetch` method can run without the need of any previous dependencies, then you can return true for `isReady` as shown in the example below. -- The `fetch` method needs to support multiple contexts in which it is called. For example, when a user requests the example of what we collect in the **Kibana>Advanced Settings>Usage data** section, the clients provided in the context of the function (`CollectorFetchContext`) are scoped to that user's privileges. The reason is to avoid exposing via telemetry any data that user should not have access to (i.e.: if the user does not have access to certain indices, they shouldn't be allowed to see the number of documents that exists in it). In this case, the `fetch` method receives the clients `esClient` and `soClient` scoped to the user who performed the HTTP API request. Alternatively, when requesting the usage data to be reported to the Remote Telemetry Service, the clients are scoped to the internal Kibana user (`kibana_system`). Please, mind it might have lower-level access than the default super-admin `elastic` test user. +- The `fetch` method needs to support multiple contexts in which it is called. For example, when a user requests the example of what we collect in the **Kibana>Advanced Settings>Usage data** section, the clients provided in the context of the function (`CollectorFetchContext`) are scoped to that user's privileges. The reason is to avoid exposing via telemetry any data that user should not have access to (i.e.: if the user does not have access to certain indices, they shouldn't be allowed to see the number of documents that exists in it). In this case, the `fetch` method receives the clients `esClient` and `soClient` scoped to the user who performed the HTTP API request. Alternatively, when requesting the usage data to be reported to the Remote Telemetry Service, the clients are scoped to the internal Kibana user (`kibana_system`). Please, mind it might have lower-level access than the default super-admin `elastic` test user. In some scenarios, your collector might need to maintain its own client. An example of that is the `monitoring` plugin, that maintains a connection to the Remote Monitoring Cluster to push its monitoring data. If that's the case, your plugin can opt-in to receive the additional `kibanaRequest` parameter by adding `extendFetchContext.kibanaRequest: true` to the collector's config: it will be appended to the context of the `fetch` method only if the request needs to be scoped to a user other than Kibana Internal, so beware that your collector will need to work for both scenarios (especially for the scenario when `kibanaRequest` is missing). Note: there will be many cases where you won't need to use the `esClient` or `soClient` function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS. @@ -502,9 +498,9 @@ There are a few ways you can test that your usage collector is working properly. ## FAQ -1. **How should I design my data model?** +1. **How should I design my data model?** Keep it simple, and keep it to a model that Kibana will be able to understand. Bear in mind the number of keys you are reporting as it may result in fields mapping explosion. Flat arrays, such as arrays of strings are fine. -2. **If I accumulate an event counter in server memory, which my fetch method returns, won't it reset when the Kibana server restarts?** +2. **If I accumulate an event counter in server memory, which my fetch method returns, won't it reset when the Kibana server restarts?** Yes, but that is not a major concern. A visualization on such info might be a date histogram that gets events-per-second or something, which would be impacted by server restarts, so we'll have to offset the beginning of the time range when we detect that the latest metric is smaller than the earliest metric. That would be a pretty custom visualization, but perhaps future Kibana enhancements will be able to support that. # Routes registered by this plugin diff --git a/src/plugins/usage_collection/server/config.ts b/src/plugins/usage_collection/server/config.ts index 87d86655ff917..eafb54fdf1339 100644 --- a/src/plugins/usage_collection/server/config.ts +++ b/src/plugins/usage_collection/server/config.ts @@ -14,7 +14,7 @@ export const configSchema = schema.object({ usageCounters: schema.object({ enabled: schema.boolean({ defaultValue: true }), retryCount: schema.number({ defaultValue: 1 }), - bufferDebounceMs: schema.number({ defaultValue: 5 * 1000 }), + bufferDurationMs: schema.duration({ defaultValue: '5s' }), }), uiCounters: schema.object({ enabled: schema.boolean({ defaultValue: true }), diff --git a/src/plugins/usage_collection/server/index.ts b/src/plugins/usage_collection/server/index.ts index 749f802a623ef..b5441a8b7b34d 100644 --- a/src/plugins/usage_collection/server/index.ts +++ b/src/plugins/usage_collection/server/index.ts @@ -22,12 +22,12 @@ export type { export type { UsageCountersSavedObject, UsageCountersSavedObjectAttributes, + IncrementCounterParams, } from './usage_counters'; export { USAGE_COUNTERS_SAVED_OBJECT_TYPE, serializeCounterKey, - deserializeCounterKey, UsageCounter, } from './usage_counters'; diff --git a/src/plugins/usage_collection/server/plugin.ts b/src/plugins/usage_collection/server/plugin.ts index 2685caad0dfd9..a12727391137e 100644 --- a/src/plugins/usage_collection/server/plugin.ts +++ b/src/plugins/usage_collection/server/plugin.ts @@ -76,7 +76,7 @@ export class UsageCollectionPlugin implements Plugin { this.usageCountersService = new UsageCountersService({ logger: this.logger.get('usage-collection', 'usage-counters-service'), retryCount: config.usageCounters.retryCount, - bufferDebounceMs: config.usageCounters.bufferDebounceMs, + bufferDurationMs: config.usageCounters.bufferDurationMs.asMilliseconds(), }); const { createUsageCounter, getUsageCounterByType } = this.usageCountersService.setup(core); diff --git a/src/plugins/usage_collection/server/usage_counters/index.ts b/src/plugins/usage_collection/server/usage_counters/index.ts index 12e2d27edef9f..dc1d1f5b43edf 100644 --- a/src/plugins/usage_collection/server/usage_counters/index.ts +++ b/src/plugins/usage_collection/server/usage_counters/index.ts @@ -8,11 +8,8 @@ export type { UsageCountersServiceSetup } from './usage_counters_service'; export type { UsageCountersSavedObjectAttributes, UsageCountersSavedObject } from './saved_objects'; +export type { IncrementCounterParams } from './usage_counter'; export { UsageCountersService } from './usage_counters_service'; export { UsageCounter } from './usage_counter'; -export { - USAGE_COUNTERS_SAVED_OBJECT_TYPE, - serializeCounterKey, - deserializeCounterKey, -} from './saved_objects'; +export { USAGE_COUNTERS_SAVED_OBJECT_TYPE, serializeCounterKey } from './saved_objects'; diff --git a/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts index 4d9c3e56d5d1d..f857d449312e6 100644 --- a/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts +++ b/src/plugins/usage_collection/server/usage_counters/saved_objects.test.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { deserializeCounterKey, serializeCounterKey, storeCounter } from './saved_objects'; +import { serializeCounterKey, storeCounter } from './saved_objects'; import { savedObjectsRepositoryMock } from '../../../../core/server/mocks'; import { CounterMetric } from './usage_counter'; import moment from 'moment'; @@ -22,17 +22,6 @@ describe('counterKey', () => { expect(result).toMatchInlineSnapshot(`"a:09042021:c:b"`); }); - - test('#deserializeCounterKey', () => { - const key = deserializeCounterKey('a:09042021:c:b'); - expect(key).toMatchInlineSnapshot(` - Object { - "counterName": "b", - "counterType": "c", - "domainId": "a", - } - `); - }); }); describe('storeCounter', () => { @@ -69,6 +58,13 @@ describe('storeCounter', () => { "incrementBy": 13, }, ], + Object { + "upsertAttributes": Object { + "counterName": "b", + "counterType": "c", + "domainId": "a", + }, + }, ] `); }); diff --git a/src/plugins/usage_collection/server/usage_counters/saved_objects.ts b/src/plugins/usage_collection/server/usage_counters/saved_objects.ts index d95b5b1fdd748..6c585d756e8c1 100644 --- a/src/plugins/usage_collection/server/usage_counters/saved_objects.ts +++ b/src/plugins/usage_collection/server/usage_counters/saved_objects.ts @@ -16,6 +16,9 @@ import moment from 'moment'; import { CounterMetric } from './usage_counter'; export interface UsageCountersSavedObjectAttributes extends SavedObjectAttributes { + domainId: string; + counterName: string; + counterType: string; count: number; } @@ -31,8 +34,9 @@ export const registerUsageCountersSavedObjectType = ( hidden: false, namespaceType: 'agnostic', mappings: { + dynamic: false, properties: { - count: { type: 'integer' }, + domainId: { type: 'keyword' }, }, }, }); @@ -55,23 +59,11 @@ export const serializeCounterKey = ({ return `${domainId}:${dayDate}:${counterType}:${counterName}`; }; -export const deserializeCounterKey = (key: string) => { - const [domainId, , counterType, ...restKey] = key.split(':'); - const counterName = restKey.join(':'); - - return { - domainId, - counterName, - counterType, - }; -}; - export const storeCounter = async ( counterMetric: CounterMetric, internalRepository: Pick ) => { const { counterName, counterType, domainId, incrementBy } = counterMetric; - const key = serializeCounterKey({ date: moment.now(), domainId, @@ -79,7 +71,16 @@ export const storeCounter = async ( counterType, }); - return await internalRepository.incrementCounter(USAGE_COUNTERS_SAVED_OBJECT_TYPE, key, [ - { fieldName: 'count', incrementBy }, - ]); + return await internalRepository.incrementCounter( + USAGE_COUNTERS_SAVED_OBJECT_TYPE, + key, + [{ fieldName: 'count', incrementBy }], + { + upsertAttributes: { + domainId, + counterName, + counterType, + }, + } + ); }; diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counter.ts b/src/plugins/usage_collection/server/usage_counters/usage_counter.ts index 78cbc7554ab1f..af00ad04149b7 100644 --- a/src/plugins/usage_collection/server/usage_counters/usage_counter.ts +++ b/src/plugins/usage_collection/server/usage_counters/usage_counter.ts @@ -20,7 +20,7 @@ export interface UsageCounterDeps { counter$: Rx.Subject; } -export interface IncrementCounterConfig { +export interface IncrementCounterParams { counterName: string; counterType?: string; incrementBy?: number; @@ -35,8 +35,8 @@ export class UsageCounter { this.counter$ = counter$; } - public incrementCounter = (config: IncrementCounterConfig) => { - const { counterName, counterType = 'count', incrementBy = 1 } = config; + public incrementCounter = (params: IncrementCounterParams) => { + const { counterName, counterType = 'count', incrementBy = 1 } = params; this.counter$.next({ counterName, diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts index 8ebfc66070581..c800bce6390c9 100644 --- a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.test.ts @@ -19,7 +19,7 @@ const tick = () => { describe('UsageCountersService', () => { const retryCount = 1; - const bufferDebounceMs = 0; + const bufferDurationMs = 100; const mockNow = 1617954426939; const logger = loggingSystemMock.createLogger(); const coreSetup = coreMock.createSetup(); @@ -34,7 +34,7 @@ describe('UsageCountersService', () => { }); it('stores data in cache during setup', async () => { - const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDebounceMs }); + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs }); const { createUsageCounter } = usageCountersService.setup(coreSetup); const usageCounter = createUsageCounter('test-counter'); @@ -49,13 +49,13 @@ describe('UsageCountersService', () => { }); it('registers savedObject type during setup', () => { - const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDebounceMs }); + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs }); usageCountersService.setup(coreSetup); expect(coreSetup.savedObjects.registerType).toBeCalledTimes(1); }); it('flushes cached data on start', async () => { - const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDebounceMs }); + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs }); const mockRepository = coreStart.savedObjects.createInternalRepository(); const mockIncrementCounter = jest.fn(); @@ -74,25 +74,25 @@ describe('UsageCountersService', () => { usageCountersService['source$'].complete(); await expect(dataInSourcePromise).resolves.toMatchInlineSnapshot(` - Array [ - Object { - "counterName": "counterA", - "counterType": "count", - "domainId": "test-counter", - "incrementBy": 1, - }, - Object { - "counterName": "counterA", - "counterType": "count", - "domainId": "test-counter", - "incrementBy": 1, - }, - ] - `); + Array [ + Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + "incrementBy": 1, + }, + Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + "incrementBy": 1, + }, + ] + `); }); it('buffers data into savedObject', async () => { - const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDebounceMs }); + const usageCountersService = new UsageCountersService({ logger, retryCount, bufferDurationMs }); const mockRepository = coreStart.savedObjects.createInternalRepository(); const mockIncrementCounter = jest.fn().mockResolvedValue('success'); @@ -122,6 +122,13 @@ describe('UsageCountersService', () => { "incrementBy": 3, }, ], + Object { + "upsertAttributes": Object { + "counterName": "counterA", + "counterType": "count", + "domainId": "test-counter", + }, + }, ], Array [ "usage-counters", @@ -132,6 +139,13 @@ describe('UsageCountersService', () => { "incrementBy": 1, }, ], + Object { + "upsertAttributes": Object { + "counterName": "counterB", + "counterType": "count", + "domainId": "test-counter", + }, + }, ], ] `); @@ -141,7 +155,7 @@ describe('UsageCountersService', () => { const usageCountersService = new UsageCountersService({ logger, retryCount: 1, - bufferDebounceMs, + bufferDurationMs, }); const mockRepository = coreStart.savedObjects.createInternalRepository(); @@ -179,11 +193,11 @@ describe('UsageCountersService', () => { ]); }); - it('buffers counters within `bufferDebounceMs` time', async () => { + it('buffers counters within `bufferDurationMs` time', async () => { const usageCountersService = new UsageCountersService({ logger, retryCount, - bufferDebounceMs: 30000, + bufferDurationMs: 30000, }); const mockRepository = coreStart.savedObjects.createInternalRepository(); diff --git a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts index d0616caf600fe..88ca9f6358926 100644 --- a/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts +++ b/src/plugins/usage_collection/server/usage_counters/usage_counters_service.ts @@ -26,7 +26,7 @@ import { export interface UsageCountersServiceDeps { logger: Logger; retryCount: number; - bufferDebounceMs: number; + bufferDurationMs: number; } export interface UsageCountersServiceSetup { @@ -47,7 +47,7 @@ export interface UsageCountersServiceStartDeps { export class UsageCountersService { private readonly stop$ = new Rx.Subject(); private readonly retryCount: number; - private readonly bufferDebounceMs: number; + private readonly bufferDurationMs: number; private readonly counterSets = new Map(); private readonly source$ = new Rx.Subject(); @@ -58,10 +58,10 @@ export class UsageCountersService { private readonly logger: Logger; - constructor({ logger, retryCount, bufferDebounceMs }: UsageCountersServiceDeps) { + constructor({ logger, retryCount, bufferDurationMs }: UsageCountersServiceDeps) { this.logger = logger; this.retryCount = retryCount; - this.bufferDebounceMs = bufferDebounceMs; + this.bufferDurationMs = bufferDurationMs; } public setup = (core: UsageCountersServiceSetupDeps): UsageCountersServiceSetup => { @@ -103,7 +103,14 @@ export class UsageCountersService { const internalRepository = savedObjects.createInternalRepository(); this.counter$ .pipe( - rxOp.buffer(this.source$.pipe(rxOp.debounceTime(this.bufferDebounceMs))), + /* buffer source events every ${bufferDurationMs} */ + rxOp.bufferTime(this.bufferDurationMs), + /** + * bufferTime will trigger every ${bufferDurationMs} + * regardless if source emitted anything or not. + * using filter will stop cut the pipe short + */ + rxOp.filter((counters) => Array.isArray(counters) && counters.length > 0), rxOp.map((counters) => Object.values(this.mergeCounters(counters))), rxOp.takeUntil(this.stop$), rxOp.concatMap((counters) => this.storeDate$(counters, internalRepository)) diff --git a/test/api_integration/config.js b/test/api_integration/config.js index e9ebe7cb0e82c..6e63fd13df8d7 100644 --- a/test/api_integration/config.js +++ b/test/api_integration/config.js @@ -32,7 +32,7 @@ export default async function ({ readConfigFile }) { '--server.compression.referrerWhitelist=["some-host.com"]', `--savedObjects.maxImportExportSize=10001`, // for testing set debounce to 0 to immediately flush counters into saved objects. - '--usageCollection.usageCounters.bufferDebounceMs=0', + '--usageCollection.usageCounters.bufferDurationMs=0', ], }, }; diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json index 3f80ca64ff4bd..80071fe422780 100644 --- a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json +++ b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/data.json @@ -80,7 +80,10 @@ "id": "uiCounter:09042021:count:myApp:some_app_event", "source": { "usage-counters": { - "count": 2 + "count": 2, + "domainId": "uiCounter", + "counterName": "myApp:some_app_event", + "counterType": "count" }, "type": "usage-counters", "updated_at": "2021-11-20T11:43:00.961Z" @@ -95,7 +98,10 @@ "id": "anotherDomainId:09042021:count:some_event_name", "source": { "usage-counters": { - "count": 2 + "count": 2, + "domainId": "anotherDomainId", + "counterName": "some_event_name", + "counterType": "count" }, "type": "usage-counters", "updated_at": "2021-11-20T11:43:00.961Z" diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json index 261e228261c64..39902f8a9211a 100644 --- a/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json +++ b/test/api_integration/fixtures/es_archiver/saved_objects/ui_counters/mappings.json @@ -36,9 +36,11 @@ } }, "usage-counters": { + "dynamic": false, "properties": { - "count": { - "type": "integer" + "domainId": { + "type": "keyword", + "ignore_above": 256 } } }, diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json index 53be9c9fc995c..16e0364b24fda 100644 --- a/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json +++ b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/data.json @@ -5,7 +5,10 @@ "id": "uiCounter:20112020:count:myApp:some_app_event", "source": { "usage-counters": { - "count": 2 + "count": 2, + "domainId": "uiCounter", + "counterName": "myApp:some_app_event", + "counterType": "count" }, "type": "usage-counters", "updated_at": "2021-11-20T11:43:00.961Z" @@ -20,7 +23,10 @@ "id": "anotherDomainId:20112020:count:some_event_name", "source": { "usage-counters": { - "count": 3 + "count": 3, + "domainId": "anotherDomainId", + "counterName": "some_event_name", + "counterType": "count" }, "type": "usage-counters", "updated_at": "2021-11-20T11:43:00.961Z" @@ -35,7 +41,10 @@ "id": "anotherDomainId:09042021:count:some_event_name", "source": { "usage-counters": { - "count": 2 + "count": 2, + "domainId": "anotherDomainId", + "counterName": "some_event_name", + "counterType": "count" }, "type": "usage-counters", "updated_at": "2021-04-09T11:43:00.961Z" @@ -50,7 +59,10 @@ "id": "anotherDomainId2:09042021:count:some_event_name", "source": { "usage-counters": { - "count": 1 + "count": 1, + "domainId": "anotherDomainId2", + "counterName": "some_event_name", + "counterType": "count" }, "type": "usage-counters", "updated_at": "2021-04-20T08:18:03.030Z" @@ -65,7 +77,10 @@ "id": "anotherDomainId3:09042021:custom_type:zero_count", "source": { "usage-counters": { - "count": 0 + "count": 0, + "domainId": "anotherDomainId3", + "counterName": "zero_count", + "counterType": "custom_type" }, "type": "usage-counters", "updated_at": "2021-04-20T08:18:03.030Z" diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json index f89299a7899f4..14ed147b2da8e 100644 --- a/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json +++ b/test/api_integration/fixtures/es_archiver/saved_objects/usage_counters/mappings.json @@ -29,9 +29,11 @@ } }, "usage-counters": { + "dynamic": false, "properties": { - "count": { - "type": "integer" + "domainId": { + "type": "keyword", + "ignore_above": 256 } } }, diff --git a/test/plugin_functional/config.ts b/test/plugin_functional/config.ts index d8bab12f7c037..c745550fdebb8 100644 --- a/test/plugin_functional/config.ts +++ b/test/plugin_functional/config.ts @@ -61,7 +61,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--corePluginDeprecations.secret=100', '--corePluginDeprecations.noLongerUsed=still_using', // for testing set debounce to 0 to immediately flush counters into saved objects. - '--usageCollection.usageCounters.bufferDebounceMs=0', + '--usageCollection.usageCounters.bufferDurationMs=0', ...plugins.map( (pluginDir) => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}` ), diff --git a/test/plugin_functional/plugins/usage_collection/kibana.json b/test/plugin_functional/plugins/usage_collection/kibana.json index d0cbc63834fec..c98e3b95d389c 100644 --- a/test/plugin_functional/plugins/usage_collection/kibana.json +++ b/test/plugin_functional/plugins/usage_collection/kibana.json @@ -3,7 +3,7 @@ "version": "0.0.1", "kibanaVersion": "kibana", "configPath": ["usageCollectionTestPlugin"], - "optionalPlugins": ["usageCollection"], + "requiredPlugins": ["usageCollection"], "server": true, "ui": false } diff --git a/test/plugin_functional/plugins/usage_collection/server/config.ts b/test/plugin_functional/plugins/usage_collection/server/config.ts deleted file mode 100644 index f79cb1fcbb74f..0000000000000 --- a/test/plugin_functional/plugins/usage_collection/server/config.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { schema, TypeOf } from '@kbn/config-schema'; -import type { PluginConfigDescriptor } from 'kibana/server'; - -const configSchema = schema.object({}); - -type ConfigType = TypeOf; - -export const config: PluginConfigDescriptor = { - schema: configSchema, -}; diff --git a/test/plugin_functional/plugins/usage_collection/server/index.ts b/test/plugin_functional/plugins/usage_collection/server/index.ts index e26e5efcc88e5..172f8491a1a40 100644 --- a/test/plugin_functional/plugins/usage_collection/server/index.ts +++ b/test/plugin_functional/plugins/usage_collection/server/index.ts @@ -7,6 +7,4 @@ */ import { UsageCollectionTestPlugin } from './plugin'; - -export { config } from './config'; export const plugin = () => new UsageCollectionTestPlugin(); diff --git a/test/plugin_functional/plugins/usage_collection/server/plugin.ts b/test/plugin_functional/plugins/usage_collection/server/plugin.ts index 32e21950d4722..523fbcfe058dc 100644 --- a/test/plugin_functional/plugins/usage_collection/server/plugin.ts +++ b/test/plugin_functional/plugins/usage_collection/server/plugin.ts @@ -14,17 +14,13 @@ import { import { registerRoutes } from './routes'; export interface TestPluginDepsSetup { - usageCollection?: UsageCollectionSetup; + usageCollection: UsageCollectionSetup; } export class UsageCollectionTestPlugin implements Plugin { private usageCounter?: UsageCounter; public setup(core: CoreSetup, { usageCollection }: TestPluginDepsSetup) { - if (!usageCollection) { - throw new Error('Optional plugin `usageCollection` is expected to be enabled for testing.'); - } - const usageCounter = usageCollection.createUsageCounter('usageCollectionTestPlugin'); registerRoutes(core.http, usageCounter); @@ -38,7 +34,7 @@ export class UsageCollectionTestPlugin implements Plugin { public start() { if (!this.usageCounter) { - throw new Error('Optional plugin `usageCollection` is expected to be enabled for testing.'); + throw new Error('this.usageCounter is expected to be defined during setup.'); } this.usageCounter.incrementCounter({ counterName: 'duringStart' }); } diff --git a/test/plugin_functional/plugins/usage_collection/server/routes.ts b/test/plugin_functional/plugins/usage_collection/server/routes.ts index 264d1b5cfbfca..e67e454512779 100644 --- a/test/plugin_functional/plugins/usage_collection/server/routes.ts +++ b/test/plugin_functional/plugins/usage_collection/server/routes.ts @@ -11,9 +11,9 @@ import { UsageCounter } from '../../../../../src/plugins/usage_collection/server export function registerRoutes(http: HttpServiceSetup, usageCounter: UsageCounter) { const router = http.createRouter(); - router.post( + router.get( { - path: '/api/usage_collection_test_plugin/', + path: '/api/usage_collection_test_plugin', validate: false, }, async (context, req, res) => { diff --git a/test/plugin_functional/test_suites/usage_collection/usage_counters.ts b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts index a5ac4324369be..f1591165b8d65 100644 --- a/test/plugin_functional/test_suites/usage_collection/usage_counters.ts +++ b/test/plugin_functional/test_suites/usage_collection/usage_counters.ts @@ -10,7 +10,6 @@ import expect from '@kbn/expect'; import { PluginFunctionalProviderContext } from '../../services'; import { UsageCountersSavedObject, - deserializeCounterKey, serializeCounterKey, } from '../../../../src/plugins/usage_collection/server/usage_counters'; @@ -28,10 +27,8 @@ export default function ({ getService, getPageObjects }: PluginFunctionalProvide .then(({ body }) => { expect(body.total).to.above(1); return (body.saved_objects as UsageCountersSavedObject[]).reduce((acc, savedObj) => { - const { counterName, domainId } = deserializeCounterKey(savedObj.id); - + const { count, counterName, domainId } = savedObj.attributes; if (domainId === 'usageCollectionTestPlugin') { - const { count } = savedObj.attributes; acc[counterName] = count; } @@ -61,10 +58,7 @@ export default function ({ getService, getPageObjects }: PluginFunctionalProvide }); it('stores usage counters triggered by runtime activities', async () => { - await supertest - .post('/api/usage_collection_test_plugin/') - .set('kbn-xsrf', 'true') - .expect(200); + await supertest.get('/api/usage_collection_test_plugin').set('kbn-xsrf', 'true').expect(200); const { routeAccessed } = await getSavedObjectCounters(); expect(routeAccessed).to.be(1); From b3baca4e8262bb2d83f5b93d59e096714634ff72 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 13 Apr 2021 20:09:10 +0300 Subject: [PATCH 08/11] type check --- .../server/collectors/usage_counters/rollups/rollups.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts index e13dab5ca72b0..c6cdaae20a8bc 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/rollups/rollups.test.ts @@ -21,9 +21,12 @@ import { USAGE_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants'; const createMockSavedObjectDoc = (updatedAt: moment.Moment, id: string) => ({ id, - type: 'ui-counter', + type: 'usage-counter', attributes: { count: 3, + counterName: 'testName', + counterType: 'count', + domainId: 'testDomain', }, references: [], updated_at: updatedAt.format(), From 333f0cd48d95688f59f740455dafc6bc4f51c5db Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 13 Apr 2021 22:17:01 +0300 Subject: [PATCH 09/11] stopUsingUiCounterIndicies$ to stop fetching ui-counters registry --- .../__fixtures__/ui_counter_saved_objects.ts | 12 +- .../usage_counter_saved_objects.ts | 11 +- .../register_ui_counters_collector.test.ts | 58 ++++++-- .../register_ui_counters_collector.ts | 133 ++++++++++-------- .../ui_counters/rollups/rollups.test.ts | 18 +-- .../collectors/ui_counters/rollups/rollups.ts | 4 +- .../usage_counter_saved_objects.ts | 7 - .../kibana_usage_collection/server/plugin.ts | 14 +- 8 files changed, 142 insertions(+), 115 deletions(-) diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts index e592e897c5861..ebc958c7be8c6 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts @@ -10,27 +10,27 @@ import type { UICounterSavedObject } from '../ui_counter_saved_object_type'; export const rawUiCounters: UICounterSavedObject[] = [ { type: 'ui-counter', - id: 'Kibana_home:24112020:click:ingest_data_card_home_tutorial_directory', + id: 'Kibana_home:23102020:click:different_type', attributes: { - count: 3, + count: 1, }, references: [], updated_at: '2020-11-24T11:27:57.067Z', - version: 'WzI5LDFd', + version: 'WzI5NDRd', }, { type: 'ui-counter', - id: 'Kibana_home:24112020:click:home_tutorial_directory', + id: 'Kibana_home:25102020:loaded:intersecting_event', attributes: { count: 1, }, references: [], - updated_at: '2020-11-24T11:27:57.067Z', + updated_at: '2020-10-25T11:27:57.067Z', version: 'WzI5NDRd', }, { type: 'ui-counter', - id: 'Kibana_home:24112020:loaded:home_tutorial_directory', + id: 'Kibana_home:23102020:loaded:intersecting_event', attributes: { count: 3, }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts index e7baca772c549..6b70a8c97e651 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/usage_counter_saved_objects.ts @@ -16,7 +16,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ count: 1, counterName: 'myApp:my_event', counterType: 'count', - timestamp: '2021-04-09T08:17:57.693Z', domainId: 'uiCounter', }, references: [], @@ -25,12 +24,11 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ }, { type: 'usage-counters', - id: 'uiCounter:09042021:loaded:Kibana_home:home_tutorial_directory', + id: 'uiCounter:23102020:loaded:Kibana_home:intersecting_event', attributes: { count: 60, - counterName: 'Kibana_home:home_tutorial_directory', + counterName: 'Kibana_home:intersecting_event', counterType: 'loaded', - timestamp: '2020-10-23T11:27:57.067Z', domainId: 'uiCounter', }, references: [], @@ -44,7 +42,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ count: 0, counterName: 'myApp:my_event_4457914848544', counterType: 'count', - timestamp: '2021-04-09T08:18:03.030Z', domainId: 'uiCounter', }, references: [], @@ -59,7 +56,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ count: 'malformed', counterName: 'myApp:my_event_malformed', counterType: 'count', - timestamp: '2021-04-09T08:18:03.030Z', domainId: 'uiCounter', }, references: [], @@ -73,7 +69,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ count: 4, counterName: 'some_event_name', counterType: 'count', - timestamp: '2021-04-09T08:18:03.030Z', domainId: 'anotherDomainId', }, references: [], @@ -87,7 +82,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ count: 8, counterName: 'myApp:my_event_4457914848544_2', counterType: 'count', - timestamp: '2021-04-09T08:18:03.031Z', domainId: 'uiCounter', }, references: [], @@ -101,7 +95,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ count: 1, counterName: 'myApp:only_reported_in_usage_counters', counterType: 'count', - timestamp: '2021-04-09T08:18:03.031Z', domainId: 'uiCounter', }, references: [], diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts index 70e6733bb6b0e..122e637d2b20c 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts @@ -9,8 +9,9 @@ import { transformRawUiCounterObject, transformRawUsageCounterObject, - fetchUiCounters, + createFetchUiCounters, } from './register_ui_counters_collector'; +import { BehaviorSubject } from 'rxjs'; import { rawUiCounters } from './__fixtures__/ui_counter_saved_objects'; import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects'; import { savedObjectsClientMock } from '../../../../../core/server/mocks'; @@ -33,7 +34,7 @@ describe('transformRawUsageCounterObject', () => { Object { "appName": "Kibana_home", "counterType": "loaded", - "eventName": "home_tutorial_directory", + "eventName": "intersecting_event", "fromTimestamp": "2020-10-23T00:00:00Z", "lastUpdatedAt": "2020-10-23T11:27:57.067Z", "total": 60, @@ -70,23 +71,23 @@ describe('transformRawUiCounterObject', () => { Object { "appName": "Kibana_home", "counterType": "click", - "eventName": "ingest_data_card_home_tutorial_directory", + "eventName": "different_type", "fromTimestamp": "2020-11-24T00:00:00Z", "lastUpdatedAt": "2020-11-24T11:27:57.067Z", - "total": 3, + "total": 1, }, Object { "appName": "Kibana_home", - "counterType": "click", - "eventName": "home_tutorial_directory", - "fromTimestamp": "2020-11-24T00:00:00Z", - "lastUpdatedAt": "2020-11-24T11:27:57.067Z", + "counterType": "loaded", + "eventName": "intersecting_event", + "fromTimestamp": "2020-10-25T00:00:00Z", + "lastUpdatedAt": "2020-10-25T11:27:57.067Z", "total": 1, }, Object { "appName": "Kibana_home", "counterType": "loaded", - "eventName": "home_tutorial_directory", + "eventName": "intersecting_event", "fromTimestamp": "2020-10-23T00:00:00Z", "lastUpdatedAt": "2020-10-23T11:27:57.067Z", "total": 3, @@ -104,11 +105,36 @@ describe('transformRawUiCounterObject', () => { }); }); -describe('fetchUiCounters', () => { +describe('createFetchUiCounters', () => { + let stopUsingUiCounterIndicies$: BehaviorSubject; const soClientMock = savedObjectsClientMock.create(); beforeEach(() => { jest.clearAllMocks(); + stopUsingUiCounterIndicies$ = new BehaviorSubject(false); }); + + it('does not query ui_counters saved objects if stopUsingUiCounterIndicies$ is complete', async () => { + // @ts-expect-error incomplete mock implementation + soClientMock.find.mockImplementation(async ({ type }) => { + switch (type) { + case USAGE_COUNTERS_SAVED_OBJECT_TYPE: + return { saved_objects: rawUsageCounters }; + default: + throw new Error(`unexpected type ${type}`); + } + }); + + stopUsingUiCounterIndicies$.complete(); + // @ts-expect-error incomplete mock implementation + const { dailyEvents } = await createFetchUiCounters(stopUsingUiCounterIndicies$)({ + soClient: soClientMock, + }); + + const transforemdUsageCounters = rawUsageCounters.map(transformRawUsageCounterObject); + expect(soClientMock.find).toBeCalledTimes(1); + expect(dailyEvents).toEqual(transforemdUsageCounters.filter(Boolean)); + }); + it('merges saved objects from both ui_counters and usage_counters saved objects', async () => { // @ts-expect-error incomplete mock implementation soClientMock.find.mockImplementation(async ({ type }) => { @@ -123,11 +149,13 @@ describe('fetchUiCounters', () => { }); // @ts-expect-error incomplete mock implementation - const { dailyEvents } = await fetchUiCounters({ soClient: soClientMock }); - expect(dailyEvents).toHaveLength(8); + const { dailyEvents } = await createFetchUiCounters(stopUsingUiCounterIndicies$)({ + soClient: soClientMock, + }); + expect(dailyEvents).toHaveLength(7); const intersectingEntry = dailyEvents.find( ({ eventName, fromTimestamp }) => - eventName === 'home_tutorial_directory' && fromTimestamp === '2020-10-23T00:00:00Z' + eventName === 'intersecting_event' && fromTimestamp === '2020-10-23T00:00:00Z' ); const onlyFromUICountersEntry = dailyEvents.find( @@ -175,10 +203,10 @@ describe('fetchUiCounters', () => { Object { "appName": "Kibana_home", "counterType": "loaded", - "eventName": "home_tutorial_directory", + "eventName": "intersecting_event", "fromTimestamp": "2020-10-23T00:00:00Z", "lastUpdatedAt": "2020-10-23T11:27:57.067Z", - "total": 60, + "total": 63, } `); }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts index 7324e49da6ec0..19190de45d96b 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts @@ -7,7 +7,8 @@ */ import moment from 'moment'; - +import { mergeWith } from 'lodash'; +import type { Subject } from 'rxjs'; import { UICounterSavedObject, UICounterSavedObjectAttributes, @@ -92,69 +93,81 @@ export function transformRawUsageCounterObject( }; } -export async function fetchUiCounters({ soClient }: CollectorFetchContext) { - const { - saved_objects: rawUsageCounters, - } = await soClient.find({ - type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, - fields: ['count', 'counterName', 'counterType', 'domainId'], - filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`, - perPage: 10000, - }); - - const { saved_objects: rawUiCounters } = await soClient.find({ - type: UI_COUNTER_SAVED_OBJECT_TYPE, - fields: ['count'], - perPage: 10000, - }); - - const dailyEventsFromUiCounters = rawUiCounters.reduce((acc, raw) => { - try { - const event = transformRawUiCounterObject(raw); - if (event) { - const { appName, eventName, counterType } = event; - const key = serializeCounterKey({ - domainId: 'uiCounters', - counterName: serializeUiCounterName({ appName, eventName }), - counterType, - date: moment(event.fromTimestamp).format('DDMMYYYY'), - }); - - acc[key] = event; +export const createFetchUiCounters = (stopUsingUiCounterIndicies$: Subject) => + async function fetchUiCounters({ soClient }: CollectorFetchContext) { + const { + saved_objects: rawUsageCounters, + } = await soClient.find({ + type: USAGE_COUNTERS_SAVED_OBJECT_TYPE, + fields: ['count', 'counterName', 'counterType', 'domainId'], + filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`, + perPage: 10000, + }); + + const skipFetchingUiCounters = stopUsingUiCounterIndicies$.isStopped; + const result = + skipFetchingUiCounters || + (await soClient.find({ + type: UI_COUNTER_SAVED_OBJECT_TYPE, + fields: ['count'], + perPage: 10000, + })); + + const rawUiCounters = typeof result === 'object' ? result.saved_objects : []; + const dailyEventsFromUiCounters = rawUiCounters.reduce((acc, raw) => { + try { + const event = transformRawUiCounterObject(raw); + if (event) { + const { appName, eventName, counterType } = event; + const key = serializeCounterKey({ + domainId: 'uiCounter', + counterName: serializeUiCounterName({ appName, eventName }), + counterType, + date: event.lastUpdatedAt, + }); + + acc[key] = event; + } + } catch (_) { + // swallow error; allows sending successfully transformed objects. } - } catch (_) { - // swallow error; allows sending successfully transformed objects. - } - return acc; - }, {} as Record); - - const dailyEventsFromUsageCounters = rawUsageCounters.reduce((acc, raw) => { - try { - const event = transformRawUsageCounterObject(raw); - if (event) { - acc[raw.id] = event; - } - } catch (_) { - // swallow error; allows sending successfully transformed objects. - } - return acc; - }, {} as Record); - - const mergedDailyCounters = Object.entries(dailyEventsFromUiCounters).reduce( - (acc, [key, value]) => { - if (acc[key]) { - value.total = acc[key].total + value.total; + return acc; + }, {} as Record); + + const dailyEventsFromUsageCounters = rawUsageCounters.reduce((acc, raw) => { + try { + const event = transformRawUsageCounterObject(raw); + if (event) { + acc[raw.id] = event; + } + } catch (_) { + // swallow error; allows sending successfully transformed objects. } - acc[key] = value; return acc; - }, - dailyEventsFromUsageCounters - ); + }, {} as Record); + + const mergedDailyCounters = mergeWith( + dailyEventsFromUsageCounters, + dailyEventsFromUiCounters, + (value: UiCounterEvent | undefined, srcValue: UiCounterEvent): UiCounterEvent => { + if (!value) { + return srcValue; + } + + return { + ...srcValue, + total: srcValue.total + value.total, + }; + } + ); - return { dailyEvents: Object.values(mergedDailyCounters) }; -} + return { dailyEvents: Object.values(mergedDailyCounters) }; + }; -export function registerUiCountersUsageCollector(usageCollection: UsageCollectionSetup) { +export function registerUiCountersUsageCollector( + usageCollection: UsageCollectionSetup, + stopUsingUiCounterIndicies$: Subject +) { const collector = usageCollection.makeUsageCollector({ type: 'ui_counters', schema: { @@ -185,7 +198,7 @@ export function registerUiCountersUsageCollector(usageCollection: UsageCollectio }, }, }, - fetch: fetchUiCounters, + fetch: createFetchUiCounters(stopUsingUiCounterIndicies$), isReady: () => true, }); diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts index 6c58884a1a259..f69ddde6a65bd 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts @@ -72,17 +72,17 @@ describe('isSavedObjectOlderThan', () => { describe('rollUiCounterIndices', () => { let logger: ReturnType; let savedObjectClient: ReturnType; - let stopRollingUiCounterIndicies$: Rx.Subject; + let stopUsingUiCounterIndicies$: Rx.Subject; beforeEach(() => { logger = loggingSystemMock.createLogger(); savedObjectClient = savedObjectsRepositoryMock.create(); - stopRollingUiCounterIndicies$ = new Rx.Subject(); + stopUsingUiCounterIndicies$ = new Rx.Subject(); }); it('returns undefined if no savedObjectsClient initialised yet', async () => { await expect( - rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, undefined) + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, undefined) ).resolves.toBe(undefined); expect(logger.warn).toHaveBeenCalledTimes(0); }); @@ -97,7 +97,7 @@ describe('rollUiCounterIndices', () => { } }); await expect( - rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) ).resolves.toEqual([]); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).not.toBeCalled(); @@ -113,9 +113,9 @@ describe('rollUiCounterIndices', () => { } }); await expect( - rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) ).resolves.toEqual([]); - expect(stopRollingUiCounterIndicies$.isStopped).toBe(true); + expect(stopUsingUiCounterIndicies$.isStopped).toBe(true); }); it(`deletes documents older than ${UI_COUNTERS_KEEP_DOCS_FOR_DAYS} days`, async () => { @@ -134,7 +134,7 @@ describe('rollUiCounterIndices', () => { } }); await expect( - rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) ).resolves.toHaveLength(2); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).toHaveBeenCalledTimes(2); @@ -156,7 +156,7 @@ describe('rollUiCounterIndices', () => { throw new Error(`Expected error!`); }); await expect( - rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) ).resolves.toEqual(undefined); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).not.toBeCalled(); @@ -178,7 +178,7 @@ describe('rollUiCounterIndices', () => { throw new Error(`Expected error!`); }); await expect( - rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, savedObjectClient) + rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient) ).resolves.toEqual(undefined); expect(savedObjectClient.find).toBeCalled(); expect(savedObjectClient.delete).toHaveBeenCalledTimes(1); diff --git a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts index 19174a648838d..79e7d3e07ba46 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts @@ -39,7 +39,7 @@ export function isSavedObjectOlderThan({ export async function rollUiCounterIndices( logger: Logger, - stopRollingUiCounterIndicies$: Subject, + stopUsingUiCounterIndicies$: Subject, savedObjectsClient?: ISavedObjectsRepository ) { if (!savedObjectsClient) { @@ -67,7 +67,7 @@ export async function rollUiCounterIndices( * migrate any docs to the Usage Counters Saved object. */ - stopRollingUiCounterIndicies$.complete(); + stopUsingUiCounterIndicies$.complete(); } const docsToDelete = rawUiCounterDocs.filter((doc) => diff --git a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts index b739744e24224..d0a45fb86b1f8 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/usage_counters/__fixtures__/usage_counter_saved_objects.ts @@ -17,7 +17,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ counterName: 'my_event', counterType: 'count', domainId: 'uiCounter', - timestamp: '2021-04-09T08:18:03.030Z', }, references: [], coreMigrationVersion: '8.0.0', @@ -31,7 +30,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ counterName: 'some_event_name', counterType: 'count', domainId: 'anotherDomainId', - timestamp: '2021-04-09T08:18:03.030Z', }, references: [], coreMigrationVersion: '8.0.0', @@ -45,7 +43,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ counterName: 'some_event_name', counterType: 'count', domainId: 'anotherDomainId', - timestamp: '2021-04-11T08:18:03.030Z', }, references: [], coreMigrationVersion: '8.0.0', @@ -59,7 +56,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ counterName: 'some_event_name', counterType: 'count', domainId: 'anotherDomainId2', - timestamp: '2021-04-20T08:18:03.030Z', }, references: [], coreMigrationVersion: '8.0.0', @@ -74,7 +70,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ counterName: 'malformed_event', counterType: 'count', domainId: 'anotherDomainId2', - timestamp: '2021-04-20T08:18:03.030Z', }, references: [], coreMigrationVersion: '8.0.0', @@ -88,7 +83,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ counterName: 'some_event_name', counterType: 'custom_type', domainId: 'anotherDomainId2', - timestamp: '2021-04-20T08:18:03.030Z', }, references: [], coreMigrationVersion: '8.0.0', @@ -102,7 +96,6 @@ export const rawUsageCounters: UsageCountersSavedObject[] = [ counterName: 'zero_count', counterType: 'custom_type', domainId: 'anotherDomainId3', - timestamp: '2021-04-20T08:18:03.030Z', }, references: [], coreMigrationVersion: '8.0.0', diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index 4ab3f5d88dffe..a27b8dff57b67 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -52,13 +52,13 @@ export class KibanaUsageCollectionPlugin implements Plugin { private uiSettingsClient?: IUiSettingsClient; private metric$: Subject; private coreUsageData?: CoreUsageDataStart; - private stopRollingUiCounterIndicies$: Subject; + private stopUsingUiCounterIndicies$: Subject; constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); this.legacyConfig$ = initializerContext.config.legacy.globalConfig$; this.metric$ = new Subject(); - this.stopRollingUiCounterIndicies$ = new Subject(); + this.stopUsingUiCounterIndicies$ = new Subject(); } public setup(coreSetup: CoreSetup, { usageCollection }: KibanaUsageCollectionPluginsDepsSetup) { @@ -68,7 +68,7 @@ export class KibanaUsageCollectionPlugin implements Plugin { usageCollection, coreSetup, this.metric$, - this.stopRollingUiCounterIndicies$, + this.stopUsingUiCounterIndicies$, coreSetup.savedObjects.registerType.bind(coreSetup.savedObjects) ); } @@ -84,14 +84,14 @@ export class KibanaUsageCollectionPlugin implements Plugin { public stop() { this.metric$.complete(); - this.stopRollingUiCounterIndicies$.complete(); + this.stopUsingUiCounterIndicies$.complete(); } private registerUsageCollectors( usageCollection: UsageCollectionSetup, coreSetup: CoreSetup, metric$: Subject, - stopRollingUiCounterIndicies$: Subject, + stopUsingUiCounterIndicies$: Subject, registerType: SavedObjectsRegisterType ) { const getSavedObjectsClient = () => this.savedObjectsClient; @@ -101,10 +101,10 @@ export class KibanaUsageCollectionPlugin implements Plugin { registerUiCounterSavedObjectType(coreSetup.savedObjects); registerUiCountersRollups( this.logger.get('ui-counters'), - stopRollingUiCounterIndicies$, + stopUsingUiCounterIndicies$, getSavedObjectsClient ); - registerUiCountersUsageCollector(usageCollection); + registerUiCountersUsageCollector(usageCollection, stopUsingUiCounterIndicies$); registerUsageCountersRollups(this.logger.get('usage-counters-rollup'), getSavedObjectsClient); registerUsageCountersUsageCollector(usageCollection); From 1163f596cc2ae993b89504a7a5c7d8d478e9d7d7 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Wed, 14 Apr 2021 10:54:02 +0300 Subject: [PATCH 10/11] Update src/plugins/usage_collection/README.mdx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Alejandro Fernández Haro --- src/plugins/usage_collection/README.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/usage_collection/README.mdx b/src/plugins/usage_collection/README.mdx index 6d53286a84dc1..a6f6f6c8e5971 100644 --- a/src/plugins/usage_collection/README.mdx +++ b/src/plugins/usage_collection/README.mdx @@ -133,7 +133,7 @@ To create a usage counter for your plugin, use the API `usageCollection.createUs ```ts // server/plugin.ts -import type { Plugin, CoreStart } from '../../../core/public'; +import type { Plugin, CoreStart } from '../../../core/server'; import type { UsageCollectionSetup, UsageCounter } from '../../../plugins/usage_collection/server'; export class MyPlugin implements Plugin { From 27370f290d51ef5f462c131f144daa39fd5ea46e Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Wed, 14 Apr 2021 13:18:54 +0300 Subject: [PATCH 11/11] config to bufferDuration --- src/plugins/usage_collection/server/config.ts | 2 +- src/plugins/usage_collection/server/plugin.ts | 2 +- test/api_integration/config.js | 4 ++-- test/plugin_functional/config.ts | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/plugins/usage_collection/server/config.ts b/src/plugins/usage_collection/server/config.ts index eafb54fdf1339..cd6f6b9d81396 100644 --- a/src/plugins/usage_collection/server/config.ts +++ b/src/plugins/usage_collection/server/config.ts @@ -14,7 +14,7 @@ export const configSchema = schema.object({ usageCounters: schema.object({ enabled: schema.boolean({ defaultValue: true }), retryCount: schema.number({ defaultValue: 1 }), - bufferDurationMs: schema.duration({ defaultValue: '5s' }), + bufferDuration: schema.duration({ defaultValue: '5s' }), }), uiCounters: schema.object({ enabled: schema.boolean({ defaultValue: true }), diff --git a/src/plugins/usage_collection/server/plugin.ts b/src/plugins/usage_collection/server/plugin.ts index a12727391137e..37d7327aed662 100644 --- a/src/plugins/usage_collection/server/plugin.ts +++ b/src/plugins/usage_collection/server/plugin.ts @@ -76,7 +76,7 @@ export class UsageCollectionPlugin implements Plugin { this.usageCountersService = new UsageCountersService({ logger: this.logger.get('usage-collection', 'usage-counters-service'), retryCount: config.usageCounters.retryCount, - bufferDurationMs: config.usageCounters.bufferDurationMs.asMilliseconds(), + bufferDurationMs: config.usageCounters.bufferDuration.asMilliseconds(), }); const { createUsageCounter, getUsageCounterByType } = this.usageCountersService.setup(core); diff --git a/test/api_integration/config.js b/test/api_integration/config.js index 6e63fd13df8d7..7bbace4c60570 100644 --- a/test/api_integration/config.js +++ b/test/api_integration/config.js @@ -31,8 +31,8 @@ export default async function ({ readConfigFile }) { '--server.xsrf.disableProtection=true', '--server.compression.referrerWhitelist=["some-host.com"]', `--savedObjects.maxImportExportSize=10001`, - // for testing set debounce to 0 to immediately flush counters into saved objects. - '--usageCollection.usageCounters.bufferDurationMs=0', + // for testing set buffer duration to 0 to immediately flush counters into saved objects. + '--usageCollection.usageCounters.bufferDuration=0', ], }, }; diff --git a/test/plugin_functional/config.ts b/test/plugin_functional/config.ts index c745550fdebb8..d21a157975ac8 100644 --- a/test/plugin_functional/config.ts +++ b/test/plugin_functional/config.ts @@ -60,8 +60,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--corePluginDeprecations.oldProperty=hello', '--corePluginDeprecations.secret=100', '--corePluginDeprecations.noLongerUsed=still_using', - // for testing set debounce to 0 to immediately flush counters into saved objects. - '--usageCollection.usageCounters.bufferDurationMs=0', + // for testing set buffer duration to 0 to immediately flush counters into saved objects. + '--usageCollection.usageCounters.bufferDuration=0', ...plugins.map( (pluginDir) => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}` ),