From cd383800e36e9f3886c80e55075f0d7a78d6fe46 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Mon, 5 Oct 2020 21:50:07 +0200 Subject: [PATCH] Core usage data (#79101) * Core Telemetry service * CoreTelemetryService mock * Add missing config values back, cleanup * Core usage collector * HttpConfig path is 'server' * Fix tests * CoreTelemetry -> CoreUsageData * Improve tests / docs * Fix telemetry_check * Don't catch fetch function exceptions, let usage collector handle it * Code review * Collect saved object index usage data * Fix tests and telemetry_check * explicitly import/export usage data types for telemetry_check * Remove OS data for now, test for SO usage data * Fix tests * Polish core docs * This shouldn't be here --- .../core_usage_data_service.mock.ts | 153 ++++++++++ .../core_usage_data_service.test.ts | 259 ++++++++++++++++ .../core_usage_data_service.ts | 285 ++++++++++++++++++ src/core/server/core_usage_data/index.ts | 31 ++ .../core_usage_data/is_configured.test.ts | 137 +++++++++ .../server/core_usage_data/is_configured.ts | 65 ++++ src/core/server/core_usage_data/types.ts | 162 ++++++++++ src/core/server/index.ts | 16 + src/core/server/internal_types.ts | 2 + src/core/server/legacy/legacy_service.ts | 5 + src/core/server/mocks.ts | 4 + src/core/server/plugins/plugin_context.ts | 1 + .../saved_objects_service.mock.ts | 11 +- src/core/server/server.api.md | 110 +++++++ src/core/server/server.ts | 10 + src/plugins/kibana_usage_collection/README.md | 1 + .../server/__snapshots__/index.test.ts.snap | 2 + .../collectors/core/core_usage_collector.ts | 132 ++++++++ .../server/collectors/core/index.test.ts | 53 ++++ .../server/collectors/core/index.ts | 20 ++ .../server/collectors/index.ts | 1 + .../kibana_usage_collection/server/plugin.ts | 6 + 22 files changed, 1462 insertions(+), 4 deletions(-) create mode 100644 src/core/server/core_usage_data/core_usage_data_service.mock.ts create mode 100644 src/core/server/core_usage_data/core_usage_data_service.test.ts create mode 100644 src/core/server/core_usage_data/core_usage_data_service.ts create mode 100644 src/core/server/core_usage_data/index.ts create mode 100644 src/core/server/core_usage_data/is_configured.test.ts create mode 100644 src/core/server/core_usage_data/is_configured.ts create mode 100644 src/core/server/core_usage_data/types.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/core/index.ts diff --git a/src/core/server/core_usage_data/core_usage_data_service.mock.ts b/src/core/server/core_usage_data/core_usage_data_service.mock.ts new file mode 100644 index 0000000000000..c443ce72f5ed7 --- /dev/null +++ b/src/core/server/core_usage_data/core_usage_data_service.mock.ts @@ -0,0 +1,153 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { PublicMethodsOf } from '@kbn/utility-types'; +import { BehaviorSubject } from 'rxjs'; +import { CoreUsageDataService } from './core_usage_data_service'; +import { CoreUsageData, CoreUsageDataStart } from './types'; + +const createStartContractMock = () => { + const startContract: jest.Mocked = { + getCoreUsageData: jest.fn().mockResolvedValue( + new BehaviorSubject({ + config: { + elasticsearch: { + apiVersion: 'master', + customHeadersConfigured: false, + healthCheckDelayMs: 2500, + logQueries: false, + numberOfHostsConfigured: 1, + pingTimeoutMs: 30000, + requestHeadersWhitelistConfigured: false, + requestTimeoutMs: 30000, + shardTimeoutMs: 30000, + sniffIntervalMs: -1, + sniffOnConnectionFault: false, + sniffOnStart: false, + ssl: { + alwaysPresentCertificate: false, + certificateAuthoritiesConfigured: false, + certificateConfigured: false, + keyConfigured: false, + verificationMode: 'full', + keystoreConfigured: false, + truststoreConfigured: false, + }, + }, + http: { + basePathConfigured: false, + compression: { + enabled: true, + referrerWhitelistConfigured: false, + }, + keepaliveTimeout: 120000, + maxPayloadInBytes: 1048576, + requestId: { + allowFromAnyIp: false, + ipAllowlistConfigured: false, + }, + rewriteBasePath: false, + socketTimeout: 120000, + ssl: { + certificateAuthoritiesConfigured: false, + certificateConfigured: false, + cipherSuites: [ + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'DHE-RSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-SHA256', + 'DHE-RSA-AES128-SHA256', + 'ECDHE-RSA-AES256-SHA384', + 'DHE-RSA-AES256-SHA384', + 'ECDHE-RSA-AES256-SHA256', + 'DHE-RSA-AES256-SHA256', + 'HIGH', + '!aNULL', + '!eNULL', + '!EXPORT', + '!DES', + '!RC4', + '!MD5', + '!PSK', + '!SRP', + '!CAMELLIA', + ], + clientAuthentication: 'none', + keyConfigured: false, + keystoreConfigured: false, + redirectHttpFromPortConfigured: false, + supportedProtocols: ['TLSv1.1', 'TLSv1.2'], + truststoreConfigured: false, + }, + xsrf: { + disableProtection: false, + whitelistConfigured: false, + }, + }, + logging: { + appendersTypesUsed: [], + loggersConfiguredCount: 0, + }, + savedObjects: { + maxImportExportSizeBytes: 10000, + maxImportPayloadBytes: 10485760, + }, + }, + environment: { + memory: { + heapSizeLimit: 1, + heapTotalBytes: 1, + heapUsedBytes: 1, + }, + }, + services: { + savedObjects: { + indices: [ + { + docsCount: 1, + docsDeleted: 1, + alias: 'test_index', + primaryStoreSizeBytes: 1, + storeSizeBytes: 1, + }, + ], + }, + }, + }) + ), + }; + + return startContract; +}; + +const createMock = () => { + const mocked: jest.Mocked> = { + setup: jest.fn(), + start: jest.fn().mockReturnValue(createStartContractMock()), + stop: jest.fn(), + }; + return mocked; +}; + +export const coreUsageDataServiceMock = { + create: createMock, + createStartContract: createStartContractMock, +}; diff --git a/src/core/server/core_usage_data/core_usage_data_service.test.ts b/src/core/server/core_usage_data/core_usage_data_service.test.ts new file mode 100644 index 0000000000000..a664f6514e9c8 --- /dev/null +++ b/src/core/server/core_usage_data/core_usage_data_service.test.ts @@ -0,0 +1,259 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { BehaviorSubject, Observable } from 'rxjs'; +import { HotObservable } from 'rxjs/internal/testing/HotObservable'; +import { TestScheduler } from 'rxjs/testing'; + +import { configServiceMock } from '../config/mocks'; + +import { mockCoreContext } from '../core_context.mock'; +import { config as RawElasticsearchConfig } from '../elasticsearch/elasticsearch_config'; +import { config as RawHttpConfig } from '../http/http_config'; +import { config as RawLoggingConfig } from '../logging/logging_config'; +import { config as RawKibanaConfig } from '../kibana_config'; +import { savedObjectsConfig as RawSavedObjectsConfig } from '../saved_objects/saved_objects_config'; +import { metricsServiceMock } from '../metrics/metrics_service.mock'; +import { savedObjectsServiceMock } from '../saved_objects/saved_objects_service.mock'; + +import { CoreUsageDataService } from './core_usage_data_service'; +import { elasticsearchServiceMock } from '../elasticsearch/elasticsearch_service.mock'; + +describe('CoreUsageDataService', () => { + const getTestScheduler = () => + new TestScheduler((actual, expected) => { + expect(actual).toEqual(expected); + }); + + let service: CoreUsageDataService; + const configService = configServiceMock.create(); + configService.atPath.mockImplementation((path) => { + if (path === 'elasticsearch') { + return new BehaviorSubject(RawElasticsearchConfig.schema.validate({})); + } else if (path === 'server') { + return new BehaviorSubject(RawHttpConfig.schema.validate({})); + } else if (path === 'logging') { + return new BehaviorSubject(RawLoggingConfig.schema.validate({})); + } else if (path === 'savedObjects') { + return new BehaviorSubject(RawSavedObjectsConfig.schema.validate({})); + } else if (path === 'kibana') { + return new BehaviorSubject(RawKibanaConfig.schema.validate({})); + } + return new BehaviorSubject({}); + }); + const coreContext = mockCoreContext.create({ configService }); + + beforeEach(() => { + service = new CoreUsageDataService(coreContext); + }); + + describe('start', () => { + describe('getCoreUsageData', () => { + it('returns core metrics for default config', () => { + const metrics = metricsServiceMock.createInternalSetupContract(); + service.setup({ metrics }); + const elasticsearch = elasticsearchServiceMock.createStart(); + elasticsearch.client.asInternalUser.cat.indices.mockResolvedValueOnce({ + body: [ + { + name: '.kibana_task_manager_1', + 'docs.count': 10, + 'docs.deleted': 10, + 'store.size': 1000, + 'pri.store.size': 2000, + }, + ], + } as any); + elasticsearch.client.asInternalUser.cat.indices.mockResolvedValueOnce({ + body: [ + { + name: '.kibana_1', + 'docs.count': 20, + 'docs.deleted': 20, + 'store.size': 2000, + 'pri.store.size': 4000, + }, + ], + } as any); + const typeRegistry = savedObjectsServiceMock.createTypeRegistryMock(); + typeRegistry.getAllTypes.mockReturnValue([ + { name: 'type 1', indexPattern: '.kibana' }, + { name: 'type 2', indexPattern: '.kibana_task_manager' }, + ] as any); + + const { getCoreUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + elasticsearch, + }); + expect(getCoreUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "config": Object { + "elasticsearch": Object { + "apiVersion": "master", + "customHeadersConfigured": false, + "healthCheckDelayMs": 2500, + "logQueries": false, + "numberOfHostsConfigured": 1, + "pingTimeoutMs": 30000, + "requestHeadersWhitelistConfigured": false, + "requestTimeoutMs": 30000, + "shardTimeoutMs": 30000, + "sniffIntervalMs": -1, + "sniffOnConnectionFault": false, + "sniffOnStart": false, + "ssl": Object { + "alwaysPresentCertificate": false, + "certificateAuthoritiesConfigured": false, + "certificateConfigured": false, + "keyConfigured": false, + "keystoreConfigured": false, + "truststoreConfigured": false, + "verificationMode": "full", + }, + }, + "http": Object { + "basePathConfigured": false, + "compression": Object { + "enabled": true, + "referrerWhitelistConfigured": false, + }, + "keepaliveTimeout": 120000, + "maxPayloadInBytes": 1048576, + "requestId": Object { + "allowFromAnyIp": false, + "ipAllowlistConfigured": false, + }, + "rewriteBasePath": false, + "socketTimeout": 120000, + "ssl": Object { + "certificateAuthoritiesConfigured": false, + "certificateConfigured": false, + "cipherSuites": Array [ + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES256-GCM-SHA384", + "DHE-RSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-SHA256", + "DHE-RSA-AES128-SHA256", + "ECDHE-RSA-AES256-SHA384", + "DHE-RSA-AES256-SHA384", + "ECDHE-RSA-AES256-SHA256", + "DHE-RSA-AES256-SHA256", + "HIGH", + "!aNULL", + "!eNULL", + "!EXPORT", + "!DES", + "!RC4", + "!MD5", + "!PSK", + "!SRP", + "!CAMELLIA", + ], + "clientAuthentication": "none", + "keyConfigured": false, + "keystoreConfigured": false, + "redirectHttpFromPortConfigured": false, + "supportedProtocols": Array [ + "TLSv1.1", + "TLSv1.2", + ], + "truststoreConfigured": false, + }, + "xsrf": Object { + "disableProtection": false, + "whitelistConfigured": false, + }, + }, + "logging": Object { + "appendersTypesUsed": Array [], + "loggersConfiguredCount": 0, + }, + "savedObjects": Object { + "maxImportExportSizeBytes": 10000, + "maxImportPayloadBytes": 10485760, + }, + }, + "environment": Object { + "memory": Object { + "heapSizeLimit": 1, + "heapTotalBytes": 1, + "heapUsedBytes": 1, + }, + }, + "services": Object { + "savedObjects": Object { + "indices": Array [ + Object { + "alias": ".kibana", + "docsCount": 10, + "docsDeleted": 10, + "primaryStoreSizeBytes": 2000, + "storeSizeBytes": 1000, + }, + Object { + "alias": ".kibana_task_manager", + "docsCount": 20, + "docsDeleted": 20, + "primaryStoreSizeBytes": 4000, + "storeSizeBytes": 2000, + }, + ], + }, + }, + } + `); + }); + }); + }); + + describe('setup and stop', () => { + it('subscribes and unsubscribes from all config paths and metrics', () => { + getTestScheduler().run(({ cold, hot, expectSubscriptions }) => { + const observables: Array> = []; + configService.atPath.mockImplementation(() => { + const newObservable = hot('-a-------'); + observables.push(newObservable); + return newObservable; + }); + const metrics = metricsServiceMock.createInternalSetupContract(); + metrics.getOpsMetrics$.mockImplementation(() => { + const newObservable = hot('-a-------'); + observables.push(newObservable); + return newObservable as Observable; + }); + + service.setup({ metrics }); + + // Use the stopTimer$ to delay calling stop() until the third frame + const stopTimer$ = cold('---a|'); + stopTimer$.subscribe(() => { + service.stop(); + }); + + const subs = '^--!'; + + observables.forEach((o) => { + expectSubscriptions(o.subscriptions).toBe(subs); + }); + }); + }); + }); +}); diff --git a/src/core/server/core_usage_data/core_usage_data_service.ts b/src/core/server/core_usage_data/core_usage_data_service.ts new file mode 100644 index 0000000000000..f729e23cb68bc --- /dev/null +++ b/src/core/server/core_usage_data/core_usage_data_service.ts @@ -0,0 +1,285 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Subject } from 'rxjs'; +import { takeUntil } from 'rxjs/operators'; + +import { CoreService } from 'src/core/types'; +import { SavedObjectsServiceStart } from 'src/core/server'; +import { CoreContext } from '../core_context'; +import { ElasticsearchConfigType } from '../elasticsearch/elasticsearch_config'; +import { HttpConfigType } from '../http'; +import { LoggingConfigType } from '../logging'; +import { SavedObjectsConfigType } from '../saved_objects/saved_objects_config'; +import { CoreServicesUsageData, CoreUsageData, CoreUsageDataStart } from './types'; +import { isConfigured } from './is_configured'; +import { ElasticsearchServiceStart } from '../elasticsearch'; +import { KibanaConfigType } from '../kibana_config'; +import { MetricsServiceSetup, OpsMetrics } from '..'; + +export interface SetupDeps { + metrics: MetricsServiceSetup; +} + +export interface StartDeps { + savedObjects: SavedObjectsServiceStart; + elasticsearch: ElasticsearchServiceStart; +} + +/** + * Because users can configure their Saved Object to any arbitrary index name, + * we need to map customized index names back to a "standard" index name. + * + * e.g. If a user configures `kibana.index: .my_saved_objects` we want to the + * collected data to be grouped under `.kibana` not ".my_saved_objects". + * + * This is rather brittle, but the option to configure index names might go + * away completely anyway (see #60053). + * + * @param index The index name configured for this SO type + * @param kibanaConfigIndex The default kibana index as configured by the user + * with `kibana.index` + */ +const kibanaOrTaskManagerIndex = (index: string, kibanaConfigIndex: string) => { + return index === kibanaConfigIndex ? '.kibana' : '.kibana_task_manager'; +}; + +export class CoreUsageDataService implements CoreService { + private elasticsearchConfig?: ElasticsearchConfigType; + private configService: CoreContext['configService']; + private httpConfig?: HttpConfigType; + private loggingConfig?: LoggingConfigType; + private soConfig?: SavedObjectsConfigType; + private stop$: Subject; + private opsMetrics?: OpsMetrics; + private kibanaConfig?: KibanaConfigType; + + constructor(core: CoreContext) { + this.configService = core.configService; + this.stop$ = new Subject(); + } + + private async getSavedObjectIndicesUsageData( + savedObjects: SavedObjectsServiceStart, + elasticsearch: ElasticsearchServiceStart + ): Promise { + const indices = await Promise.all( + Array.from( + savedObjects + .getTypeRegistry() + .getAllTypes() + .reduce((acc, type) => { + const index = type.indexPattern ?? this.kibanaConfig!.index; + return index != null ? acc.add(index) : acc; + }, new Set()) + .values() + ).map((index) => { + // The _cat/indices API returns the _index_ and doesn't return a way + // to map back from the index to the alias. So we have to make an API + // call for every alias + return elasticsearch.client.asInternalUser.cat + .indices({ + index, + format: 'JSON', + bytes: 'b', + }) + .then(({ body }) => { + const stats = body[0]; + return { + alias: kibanaOrTaskManagerIndex(index, this.kibanaConfig!.index), + docsCount: stats['docs.count'], + docsDeleted: stats['docs.deleted'], + storeSizeBytes: stats['store.size'], + primaryStoreSizeBytes: stats['pri.store.size'], + }; + }); + }) + ); + + return { + indices, + }; + } + + private async getCoreUsageData( + savedObjects: SavedObjectsServiceStart, + elasticsearch: ElasticsearchServiceStart + ): Promise { + if ( + this.elasticsearchConfig == null || + this.httpConfig == null || + this.soConfig == null || + this.opsMetrics == null + ) { + throw new Error('Unable to read config values. Ensure that setup() has completed.'); + } + + const es = this.elasticsearchConfig; + const soUsageData = await this.getSavedObjectIndicesUsageData(savedObjects, elasticsearch); + + const http = this.httpConfig; + return { + config: { + elasticsearch: { + apiVersion: es.apiVersion, + sniffOnStart: es.sniffOnStart, + sniffIntervalMs: es.sniffInterval !== false ? es.sniffInterval.asMilliseconds() : -1, + sniffOnConnectionFault: es.sniffOnConnectionFault, + numberOfHostsConfigured: Array.isArray(es.hosts) + ? es.hosts.length + : isConfigured.string(es.hosts) + ? 1 + : 0, + customHeadersConfigured: isConfigured.record(es.customHeaders), + healthCheckDelayMs: es.healthCheck.delay.asMilliseconds(), + logQueries: es.logQueries, + pingTimeoutMs: es.pingTimeout.asMilliseconds(), + requestHeadersWhitelistConfigured: isConfigured.stringOrArray( + es.requestHeadersWhitelist, + ['authorization'] + ), + requestTimeoutMs: es.requestTimeout.asMilliseconds(), + shardTimeoutMs: es.shardTimeout.asMilliseconds(), + ssl: { + alwaysPresentCertificate: es.ssl.alwaysPresentCertificate, + certificateAuthoritiesConfigured: isConfigured.stringOrArray( + es.ssl.certificateAuthorities + ), + certificateConfigured: isConfigured.string(es.ssl.certificate), + keyConfigured: isConfigured.string(es.ssl.key), + verificationMode: es.ssl.verificationMode, + truststoreConfigured: isConfigured.record(es.ssl.truststore), + keystoreConfigured: isConfigured.record(es.ssl.keystore), + }, + }, + http: { + basePathConfigured: isConfigured.string(http.basePath), + maxPayloadInBytes: http.maxPayload.getValueInBytes(), + rewriteBasePath: http.rewriteBasePath, + keepaliveTimeout: http.keepaliveTimeout, + socketTimeout: http.socketTimeout, + compression: { + enabled: http.compression.enabled, + referrerWhitelistConfigured: isConfigured.array(http.compression.referrerWhitelist), + }, + xsrf: { + disableProtection: http.xsrf.disableProtection, + whitelistConfigured: isConfigured.array(http.xsrf.whitelist), + }, + requestId: { + allowFromAnyIp: http.requestId.allowFromAnyIp, + ipAllowlistConfigured: isConfigured.array(http.requestId.ipAllowlist), + }, + ssl: { + certificateAuthoritiesConfigured: isConfigured.stringOrArray( + http.ssl.certificateAuthorities + ), + certificateConfigured: isConfigured.string(http.ssl.certificate), + cipherSuites: http.ssl.cipherSuites, + keyConfigured: isConfigured.string(http.ssl.key), + redirectHttpFromPortConfigured: isConfigured.number(http.ssl.redirectHttpFromPort), + supportedProtocols: http.ssl.supportedProtocols, + clientAuthentication: http.ssl.clientAuthentication, + keystoreConfigured: isConfigured.record(http.ssl.keystore), + truststoreConfigured: isConfigured.record(http.ssl.truststore), + }, + }, + + logging: { + appendersTypesUsed: Array.from( + Array.from(this.loggingConfig?.appenders.values() ?? []) + .reduce((acc, a) => acc.add(a.kind), new Set()) + .values() + ), + loggersConfiguredCount: this.loggingConfig?.loggers.length ?? 0, + }, + + savedObjects: { + maxImportPayloadBytes: this.soConfig.maxImportPayloadBytes.getValueInBytes(), + maxImportExportSizeBytes: this.soConfig.maxImportExportSize.getValueInBytes(), + }, + }, + environment: { + memory: { + heapSizeLimit: this.opsMetrics.process.memory.heap.size_limit, + heapTotalBytes: this.opsMetrics.process.memory.heap.total_in_bytes, + heapUsedBytes: this.opsMetrics.process.memory.heap.used_in_bytes, + }, + }, + services: { + savedObjects: soUsageData, + }, + }; + } + + setup({ metrics }: SetupDeps) { + metrics + .getOpsMetrics$() + .pipe(takeUntil(this.stop$)) + .subscribe((opsMetrics) => (this.opsMetrics = opsMetrics)); + + this.configService + .atPath('elasticsearch') + .pipe(takeUntil(this.stop$)) + .subscribe((config) => { + this.elasticsearchConfig = config; + }); + + this.configService + .atPath('server') + .pipe(takeUntil(this.stop$)) + .subscribe((config) => { + this.httpConfig = config; + }); + + this.configService + .atPath('logging') + .pipe(takeUntil(this.stop$)) + .subscribe((config) => { + this.loggingConfig = config; + }); + + this.configService + .atPath('savedObjects') + .pipe(takeUntil(this.stop$)) + .subscribe((config) => { + this.soConfig = config; + }); + + this.configService + .atPath('kibana') + .pipe(takeUntil(this.stop$)) + .subscribe((config) => { + this.kibanaConfig = config; + }); + } + + start({ savedObjects, elasticsearch }: StartDeps) { + return { + getCoreUsageData: () => { + return this.getCoreUsageData(savedObjects, elasticsearch); + }, + }; + } + + stop() { + this.stop$.next(); + this.stop$.complete(); + } +} diff --git a/src/core/server/core_usage_data/index.ts b/src/core/server/core_usage_data/index.ts new file mode 100644 index 0000000000000..b78c126657ef6 --- /dev/null +++ b/src/core/server/core_usage_data/index.ts @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export { CoreUsageDataStart } from './types'; +export { CoreUsageDataService } from './core_usage_data_service'; + +// Because of #79265 we need to explicity import, then export these types for +// scripts/telemetry_check.js to work as expected +import { + CoreUsageData, + CoreConfigUsageData, + CoreEnvironmentUsageData, + CoreServicesUsageData, +} from './types'; + +export { CoreUsageData, CoreConfigUsageData, CoreEnvironmentUsageData, CoreServicesUsageData }; diff --git a/src/core/server/core_usage_data/is_configured.test.ts b/src/core/server/core_usage_data/is_configured.test.ts new file mode 100644 index 0000000000000..e5d04946b8766 --- /dev/null +++ b/src/core/server/core_usage_data/is_configured.test.ts @@ -0,0 +1,137 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { isConfigured } from './is_configured'; + +describe('isConfigured', () => { + describe('#string', () => { + it('returns true for a non-empty string', () => { + expect(isConfigured.string('I am configured')).toEqual(true); + }); + + it('returns false for an empty string', () => { + expect(isConfigured.string(' ')).toEqual(false); + expect(isConfigured.string(' ')).toEqual(false); + }); + + it('returns false for undefined', () => { + expect(isConfigured.string(undefined)).toEqual(false); + }); + + it('returns false for null', () => { + expect(isConfigured.string(null as any)).toEqual(false); + }); + + it('returns false for a record', () => { + expect(isConfigured.string({} as any)).toEqual(false); + expect(isConfigured.string({ key: 'hello' } as any)).toEqual(false); + }); + + it('returns false for an array', () => { + expect(isConfigured.string([] as any)).toEqual(false); + expect(isConfigured.string(['hello'] as any)).toEqual(false); + }); + }); + + describe('array', () => { + it('returns true for a non-empty array', () => { + expect(isConfigured.array(['a'])).toEqual(true); + expect(isConfigured.array([{}])).toEqual(true); + expect(isConfigured.array([{ key: 'hello' }])).toEqual(true); + }); + + it('returns false for an empty array', () => { + expect(isConfigured.array([])).toEqual(false); + }); + + it('returns false for undefined', () => { + expect(isConfigured.array(undefined)).toEqual(false); + }); + + it('returns false for null', () => { + expect(isConfigured.array(null as any)).toEqual(false); + }); + + it('returns false for a string', () => { + expect(isConfigured.array('string')).toEqual(false); + }); + + it('returns false for a record', () => { + expect(isConfigured.array({} as any)).toEqual(false); + }); + }); + + describe('stringOrArray', () => { + const arraySpy = jest.spyOn(isConfigured, 'array'); + const stringSpy = jest.spyOn(isConfigured, 'string'); + + it('calls #array for an array', () => { + isConfigured.stringOrArray([]); + expect(arraySpy).toHaveBeenCalledWith([]); + }); + + it('calls #string for non-array values', () => { + isConfigured.stringOrArray('string'); + expect(stringSpy).toHaveBeenCalledWith('string'); + }); + }); + + describe('record', () => { + it('returns true for a non-empty record', () => { + expect(isConfigured.record({ key: 'hello' })).toEqual(true); + expect(isConfigured.record({ key: undefined })).toEqual(true); + }); + + it('returns false for an empty record', () => { + expect(isConfigured.record({})).toEqual(false); + }); + it('returns false for undefined', () => { + expect(isConfigured.record(undefined)).toEqual(false); + }); + it('returns false for null', () => { + expect(isConfigured.record(null as any)).toEqual(false); + }); + }); + + describe('number', () => { + it('returns true for a valid number', () => { + expect(isConfigured.number(0)).toEqual(true); + expect(isConfigured.number(-0)).toEqual(true); + expect(isConfigured.number(1)).toEqual(true); + expect(isConfigured.number(-0)).toEqual(true); + }); + + it('returns false for NaN', () => { + expect(isConfigured.number(Number.NaN)).toEqual(false); + }); + + it('returns false for a string', () => { + expect(isConfigured.number('1' as any)).toEqual(false); + expect(isConfigured.number('' as any)).toEqual(false); + }); + + it('returns false for undefined', () => { + expect(isConfigured.number(undefined)).toEqual(false); + }); + + it('returns false for null', () => { + expect(isConfigured.number(null as any)).toEqual(false); + }); + }); +}); diff --git a/src/core/server/core_usage_data/is_configured.ts b/src/core/server/core_usage_data/is_configured.ts new file mode 100644 index 0000000000000..e66f990f1037c --- /dev/null +++ b/src/core/server/core_usage_data/is_configured.ts @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { isEqual } from 'lodash'; + +/** + * Test whether a given config value is configured based on it's schema type. + * Our configuration schema and code often accept and ignore empty values like + * `elasticsearch.customHeaders: {}`. However, for telemetry purposes, we're + * only interested when these values have been set to something that will + * change the behaviour of Kibana. + */ +export const isConfigured = { + /** + * config is a string with non-zero length + */ + string: (config?: string): boolean => { + return (config?.trim?.()?.length ?? 0) > 0; + }, + /** + * config is an array with non-zero length + */ + array: (config?: unknown[] | string, defaultValue?: any): boolean => { + return Array.isArray(config) + ? (config?.length ?? 0) > 0 && !isEqual(config, defaultValue) + : false; + }, + /** + * config is a string or array of strings where each element has non-zero length + */ + stringOrArray: (config?: string[] | string, defaultValue?: any): boolean => { + return Array.isArray(config) + ? isConfigured.array(config, defaultValue) + : isConfigured.string(config); + }, + /** + * config is a record with at least one key + */ + record: (config?: Record): boolean => { + return config != null && typeof config === 'object' && Object.keys(config).length > 0; + }, + /** + * config is a number + */ + number: (config?: number): boolean => { + // kbn-config-schema already does NaN validation, but doesn't hurt to be sure + return typeof config === 'number' && !isNaN(config); + }, +}; diff --git a/src/core/server/core_usage_data/types.ts b/src/core/server/core_usage_data/types.ts new file mode 100644 index 0000000000000..52d2eadcf1377 --- /dev/null +++ b/src/core/server/core_usage_data/types.ts @@ -0,0 +1,162 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Type describing Core's usage data payload + * @internal + */ +export interface CoreUsageData { + config: CoreConfigUsageData; + services: CoreServicesUsageData; + environment: CoreEnvironmentUsageData; +} + +/** + * Usage data from Core services + * @internal + */ +export interface CoreServicesUsageData { + savedObjects: { + // scripts/telemetry_check.js does not support parsing Array<{...}> types + // so we have to disable eslint here and use {...}[] + // eslint-disable-next-line @typescript-eslint/array-type + indices: { + alias: string; + docsCount: number; + docsDeleted: number; + storeSizeBytes: number; + primaryStoreSizeBytes: number; + }[]; + }; +} + +/** + * Usage data on this Kibana node's runtime environment. + * @internal + */ +export interface CoreEnvironmentUsageData { + memory: { + heapTotalBytes: number; + heapUsedBytes: number; + /** V8 heap size limit */ + heapSizeLimit: number; + }; +} + +/** + * Usage data on this cluster's configuration of Core features + * @internal + */ +export interface CoreConfigUsageData { + elasticsearch: { + sniffOnStart: boolean; + sniffIntervalMs?: number; + sniffOnConnectionFault: boolean; + numberOfHostsConfigured: number; + requestHeadersWhitelistConfigured: boolean; + customHeadersConfigured: boolean; + shardTimeoutMs: number; + requestTimeoutMs: number; + pingTimeoutMs: number; + logQueries: boolean; + ssl: { + verificationMode: 'none' | 'certificate' | 'full'; + certificateAuthoritiesConfigured: boolean; + certificateConfigured: boolean; + keyConfigured: boolean; + keystoreConfigured: boolean; + truststoreConfigured: boolean; + alwaysPresentCertificate: boolean; + }; + apiVersion: string; + healthCheckDelayMs: number; + }; + + http: { + basePathConfigured: boolean; + maxPayloadInBytes: number; + rewriteBasePath: boolean; + keepaliveTimeout: number; + socketTimeout: number; + compression: { + enabled: boolean; + referrerWhitelistConfigured: boolean; + }; + xsrf: { + disableProtection: boolean; + whitelistConfigured: boolean; + }; + requestId: { + allowFromAnyIp: boolean; + ipAllowlistConfigured: boolean; + }; + ssl: { + certificateAuthoritiesConfigured: boolean; + certificateConfigured: boolean; + cipherSuites: string[]; + keyConfigured: boolean; + keystoreConfigured: boolean; + truststoreConfigured: boolean; + redirectHttpFromPortConfigured: boolean; + supportedProtocols: string[]; + clientAuthentication: 'none' | 'optional' | 'required'; + }; + }; + + logging: { + appendersTypesUsed: string[]; + loggersConfiguredCount: number; + }; + + // plugins: { + // /** list of built-in plugins that are disabled */ + // firstPartyDisabled: string[]; + // /** list of third-party plugins that are installed and enabled */ + // thirdParty: string[]; + // }; + + savedObjects: { + maxImportPayloadBytes: number; + maxImportExportSizeBytes: number; + }; + + // uiSettings: { + // overridesCount: number; + // }; +} + +/** + * Internal API for getting Core's usage data payload. + * + * @note This API should never be used to drive application logic and is only + * intended for telemetry purposes. + * + * @internal + */ +export interface CoreUsageDataStart { + /** + * Internal API for getting Core's usage data payload. + * + * @note This API should never be used to drive application logic and is only + * intended for telemetry purposes. + * + * @internal + * */ + getCoreUsageData(): Promise; +} diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 70ef93963c69f..887dc50d5f78b 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -64,6 +64,18 @@ import { MetricsServiceSetup, MetricsServiceStart } from './metrics'; import { StatusServiceSetup } from './status'; import { Auditor, AuditTrailSetup, AuditTrailStart } from './audit_trail'; import { AppenderConfigType, appendersSchema, LoggingServiceSetup } from './logging'; +import { CoreUsageDataStart } from './core_usage_data'; + +// Because of #79265 we need to explicity import, then export these types for +// scripts/telemetry_check.js to work as expected +import { + CoreUsageData, + CoreConfigUsageData, + CoreEnvironmentUsageData, + CoreServicesUsageData, +} from './core_usage_data'; + +export { CoreUsageData, CoreConfigUsageData, CoreEnvironmentUsageData, CoreServicesUsageData }; export { AuditableEvent, Auditor, AuditorFactory, AuditTrailSetup } from './audit_trail'; export { bootstrap } from './bootstrap'; @@ -349,6 +361,8 @@ export { StatusServiceSetup, } from './status'; +export { CoreUsageDataStart } from './core_usage_data'; + /** * Plugin specific context passed to a route handler. * @@ -456,6 +470,8 @@ export interface CoreStart { uiSettings: UiSettingsServiceStart; /** {@link AuditTrailSetup} */ auditTrail: AuditTrailStart; + /** @internal {@link CoreUsageDataStart} */ + coreUsageData: CoreUsageDataStart; } export { diff --git a/src/core/server/internal_types.ts b/src/core/server/internal_types.ts index f5a5edffb0a74..ce58348a14153 100644 --- a/src/core/server/internal_types.ts +++ b/src/core/server/internal_types.ts @@ -39,6 +39,7 @@ import { InternalHttpResourcesSetup } from './http_resources'; import { InternalStatusServiceSetup } from './status'; import { AuditTrailSetup, AuditTrailStart } from './audit_trail'; import { InternalLoggingServiceSetup } from './logging'; +import { CoreUsageDataStart } from './core_usage_data'; /** @internal */ export interface InternalCoreSetup { @@ -68,6 +69,7 @@ export interface InternalCoreStart { savedObjects: InternalSavedObjectsServiceStart; uiSettings: InternalUiSettingsServiceStart; auditTrail: AuditTrailStart; + coreUsageData: CoreUsageDataStart; } /** diff --git a/src/core/server/legacy/legacy_service.ts b/src/core/server/legacy/legacy_service.ts index 086e20c98c1a3..75e8ae6524920 100644 --- a/src/core/server/legacy/legacy_service.ts +++ b/src/core/server/legacy/legacy_service.ts @@ -217,6 +217,11 @@ export class LegacyService implements CoreService { }, uiSettings: { asScopedToClient: startDeps.core.uiSettings.asScopedToClient }, auditTrail: startDeps.core.auditTrail, + coreUsageData: { + getCoreUsageData: () => { + throw new Error('core.start.coreUsageData.getCoreUsageData is unsupported in legacy'); + }, + }, }; const router = setupDeps.core.http.createRouter('', this.legacyId); diff --git a/src/core/server/mocks.ts b/src/core/server/mocks.ts index 3030cd9f4e0cb..34e85920efb24 100644 --- a/src/core/server/mocks.ts +++ b/src/core/server/mocks.ts @@ -37,6 +37,7 @@ import { metricsServiceMock } from './metrics/metrics_service.mock'; import { environmentServiceMock } from './environment/environment_service.mock'; import { statusServiceMock } from './status/status_service.mock'; import { auditTrailServiceMock } from './audit_trail/audit_trail_service.mock'; +import { coreUsageDataServiceMock } from './core_usage_data/core_usage_data_service.mock'; export { configServiceMock } from './config/mocks'; export { httpServerMock } from './http/http_server.mocks'; @@ -55,6 +56,7 @@ export { renderingMock } from './rendering/rendering_service.mock'; export { statusServiceMock } from './status/status_service.mock'; export { contextServiceMock } from './context/context_service.mock'; export { capabilitiesServiceMock } from './capabilities/capabilities_service.mock'; +export { coreUsageDataServiceMock } from './core_usage_data/core_usage_data_service.mock'; export function pluginInitializerContextConfigMock(config: T) { const globalConfig: SharedGlobalConfig = { @@ -157,6 +159,7 @@ function createCoreStartMock() { metrics: metricsServiceMock.createStartContract(), savedObjects: savedObjectsServiceMock.createStartContract(), uiSettings: uiSettingsServiceMock.createStartContract(), + coreUsageData: coreUsageDataServiceMock.createStartContract(), }; return mock; @@ -190,6 +193,7 @@ function createInternalCoreStartMock() { savedObjects: savedObjectsServiceMock.createInternalStartContract(), uiSettings: uiSettingsServiceMock.createStartContract(), auditTrail: auditTrailServiceMock.createStartContract(), + coreUsageData: coreUsageDataServiceMock.createStartContract(), }; return startDeps; } diff --git a/src/core/server/plugins/plugin_context.ts b/src/core/server/plugins/plugin_context.ts index ab3f471fd7942..a8249ed7e3218 100644 --- a/src/core/server/plugins/plugin_context.ts +++ b/src/core/server/plugins/plugin_context.ts @@ -251,5 +251,6 @@ export function createPluginStartContext( asScopedToClient: deps.uiSettings.asScopedToClient, }, auditTrail: deps.auditTrail, + coreUsageData: deps.coreUsageData, }; } diff --git a/src/core/server/saved_objects/saved_objects_service.mock.ts b/src/core/server/saved_objects/saved_objects_service.mock.ts index bd76658c21731..c56cdabf6e4cd 100644 --- a/src/core/server/saved_objects/saved_objects_service.mock.ts +++ b/src/core/server/saved_objects/saved_objects_service.mock.ts @@ -33,10 +33,11 @@ import { savedObjectsClientMock } from './service/saved_objects_client.mock'; import { typeRegistryMock } from './saved_objects_type_registry.mock'; import { migrationMocks } from './migrations/mocks'; import { ServiceStatusLevels } from '../status'; +import { ISavedObjectTypeRegistry } from './saved_objects_type_registry'; type SavedObjectsServiceContract = PublicMethodsOf; -const createStartContractMock = () => { +const createStartContractMock = (typeRegistry?: jest.Mocked) => { const startContrat: jest.Mocked = { getScopedClient: jest.fn(), createInternalRepository: jest.fn(), @@ -48,13 +49,15 @@ const createStartContractMock = () => { startContrat.getScopedClient.mockReturnValue(savedObjectsClientMock.create()); startContrat.createInternalRepository.mockReturnValue(savedObjectsRepositoryMock.create()); startContrat.createScopedRepository.mockReturnValue(savedObjectsRepositoryMock.create()); - startContrat.getTypeRegistry.mockReturnValue(typeRegistryMock.create()); + startContrat.getTypeRegistry.mockReturnValue(typeRegistry ?? typeRegistryMock.create()); return startContrat; }; -const createInternalStartContractMock = () => { - const internalStartContract: jest.Mocked = createStartContractMock(); +const createInternalStartContractMock = (typeRegistry?: jest.Mocked) => { + const internalStartContract: jest.Mocked = createStartContractMock( + typeRegistry + ); return internalStartContract; }; diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 7742dad150cfa..a718ae8a6ff17 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -401,9 +401,102 @@ export interface ContextSetup { createContextContainer>(): IContextContainer; } +// @internal +export interface CoreConfigUsageData { + // (undocumented) + elasticsearch: { + sniffOnStart: boolean; + sniffIntervalMs?: number; + sniffOnConnectionFault: boolean; + numberOfHostsConfigured: number; + requestHeadersWhitelistConfigured: boolean; + customHeadersConfigured: boolean; + shardTimeoutMs: number; + requestTimeoutMs: number; + pingTimeoutMs: number; + logQueries: boolean; + ssl: { + verificationMode: 'none' | 'certificate' | 'full'; + certificateAuthoritiesConfigured: boolean; + certificateConfigured: boolean; + keyConfigured: boolean; + keystoreConfigured: boolean; + truststoreConfigured: boolean; + alwaysPresentCertificate: boolean; + }; + apiVersion: string; + healthCheckDelayMs: number; + }; + // (undocumented) + http: { + basePathConfigured: boolean; + maxPayloadInBytes: number; + rewriteBasePath: boolean; + keepaliveTimeout: number; + socketTimeout: number; + compression: { + enabled: boolean; + referrerWhitelistConfigured: boolean; + }; + xsrf: { + disableProtection: boolean; + whitelistConfigured: boolean; + }; + requestId: { + allowFromAnyIp: boolean; + ipAllowlistConfigured: boolean; + }; + ssl: { + certificateAuthoritiesConfigured: boolean; + certificateConfigured: boolean; + cipherSuites: string[]; + keyConfigured: boolean; + keystoreConfigured: boolean; + truststoreConfigured: boolean; + redirectHttpFromPortConfigured: boolean; + supportedProtocols: string[]; + clientAuthentication: 'none' | 'optional' | 'required'; + }; + }; + // (undocumented) + logging: { + appendersTypesUsed: string[]; + loggersConfiguredCount: number; + }; + // (undocumented) + savedObjects: { + maxImportPayloadBytes: number; + maxImportExportSizeBytes: number; + }; +} + +// @internal +export interface CoreEnvironmentUsageData { + // (undocumented) + memory: { + heapTotalBytes: number; + heapUsedBytes: number; + heapSizeLimit: number; + }; +} + // @internal (undocumented) export type CoreId = symbol; +// @internal +export interface CoreServicesUsageData { + // (undocumented) + savedObjects: { + indices: { + alias: string; + docsCount: number; + docsDeleted: number; + storeSizeBytes: number; + primaryStoreSizeBytes: number; + }[]; + }; +} + // @public export interface CoreSetup { // (undocumented) @@ -438,6 +531,8 @@ export interface CoreStart { auditTrail: AuditTrailStart; // (undocumented) capabilities: CapabilitiesStart; + // @internal (undocumented) + coreUsageData: CoreUsageDataStart; // (undocumented) elasticsearch: ElasticsearchServiceStart; // (undocumented) @@ -458,6 +553,21 @@ export interface CoreStatus { savedObjects: ServiceStatus; } +// @internal +export interface CoreUsageData { + // (undocumented) + config: CoreConfigUsageData; + // (undocumented) + environment: CoreEnvironmentUsageData; + // (undocumented) + services: CoreServicesUsageData; +} + +// @internal +export interface CoreUsageDataStart { + getCoreUsageData(): Promise; +} + // @public (undocumented) export interface CountResponse { // (undocumented) diff --git a/src/core/server/server.ts b/src/core/server/server.ts index 600f45e0b50da..f38cac4f43768 100644 --- a/src/core/server/server.ts +++ b/src/core/server/server.ts @@ -48,6 +48,7 @@ import { config as statusConfig } from './status'; import { ContextService } from './context'; import { RequestHandlerContext } from '.'; import { InternalCoreSetup, InternalCoreStart, ServiceConfigDescriptor } from './internal_types'; +import { CoreUsageDataService } from './core_usage_data'; import { CoreRouteHandlerContext } from './core_route_handler_context'; const coreId = Symbol('core'); @@ -72,6 +73,7 @@ export class Server { private readonly logging: LoggingService; private readonly coreApp: CoreApp; private readonly auditTrail: AuditTrailService; + private readonly coreUsageData: CoreUsageDataService; #pluginsInitialized?: boolean; private coreStart?: InternalCoreStart; @@ -103,6 +105,7 @@ export class Server { this.httpResources = new HttpResourcesService(core); this.auditTrail = new AuditTrailService(core); this.logging = new LoggingService(core); + this.coreUsageData = new CoreUsageDataService(core); } public async setup() { @@ -184,6 +187,8 @@ export class Server { loggingSystem: this.loggingSystem, }); + this.coreUsageData.setup({ metrics: metricsSetup }); + const coreSetup: InternalCoreSetup = { capabilities: capabilitiesSetup, context: contextServiceSetup, @@ -235,6 +240,10 @@ export class Server { const uiSettingsStart = await this.uiSettings.start(); const metricsStart = await this.metrics.start(); const httpStart = this.http.getStartContract(); + const coreUsageDataStart = this.coreUsageData.start({ + elasticsearch: elasticsearchStart, + savedObjects: savedObjectsStart, + }); this.coreStart = { capabilities: capabilitiesStart, @@ -244,6 +253,7 @@ export class Server { savedObjects: savedObjectsStart, uiSettings: uiSettingsStart, auditTrail: auditTrailStart, + coreUsageData: coreUsageDataStart, }; const pluginsStart = await this.plugins.start(this.coreStart); diff --git a/src/plugins/kibana_usage_collection/README.md b/src/plugins/kibana_usage_collection/README.md index 73a4d53f305f2..69711d30cdc74 100644 --- a/src/plugins/kibana_usage_collection/README.md +++ b/src/plugins/kibana_usage_collection/README.md @@ -8,3 +8,4 @@ This plugin registers the basic usage collectors from Kibana: - Number of Saved Objects per type - Non-default UI Settings - CSP configuration +- Core Metrics diff --git a/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap b/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap index 47a4c458a8398..c479562795512 100644 --- a/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap +++ b/src/plugins/kibana_usage_collection/server/__snapshots__/index.test.ts.snap @@ -11,3 +11,5 @@ exports[`kibana_usage_collection Runs the setup method without issues 4`] = `fal exports[`kibana_usage_collection Runs the setup method without issues 5`] = `false`; exports[`kibana_usage_collection Runs the setup method without issues 6`] = `true`; + +exports[`kibana_usage_collection Runs the setup method without issues 7`] = `false`; diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts new file mode 100644 index 0000000000000..297baf016e9e6 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts @@ -0,0 +1,132 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { CoreUsageData, CoreUsageDataStart } from '../../../../../core/server'; + +export function getCoreUsageCollector( + usageCollection: UsageCollectionSetup, + getCoreUsageDataService: () => CoreUsageDataStart +) { + return usageCollection.makeUsageCollector({ + type: 'core', + isReady: () => typeof getCoreUsageDataService() !== 'undefined', + schema: { + config: { + elasticsearch: { + sniffOnStart: { type: 'boolean' }, + sniffIntervalMs: { type: 'long' }, + sniffOnConnectionFault: { type: 'boolean' }, + numberOfHostsConfigured: { type: 'long' }, + requestHeadersWhitelistConfigured: { type: 'boolean' }, + customHeadersConfigured: { type: 'boolean' }, + shardTimeoutMs: { type: 'long' }, + requestTimeoutMs: { type: 'long' }, + pingTimeoutMs: { type: 'long' }, + logQueries: { type: 'boolean' }, + ssl: { + verificationMode: { type: 'keyword' }, + certificateAuthoritiesConfigured: { type: 'boolean' }, + certificateConfigured: { type: 'boolean' }, + keyConfigured: { type: 'boolean' }, + keystoreConfigured: { type: 'boolean' }, + truststoreConfigured: { type: 'boolean' }, + alwaysPresentCertificate: { type: 'boolean' }, + }, + apiVersion: { type: 'keyword' }, + healthCheckDelayMs: { type: 'long' }, + }, + + http: { + basePathConfigured: { type: 'boolean' }, + maxPayloadInBytes: { type: 'long' }, + rewriteBasePath: { type: 'boolean' }, + keepaliveTimeout: { type: 'long' }, + socketTimeout: { type: 'long' }, + compression: { + enabled: { type: 'boolean' }, + referrerWhitelistConfigured: { type: 'boolean' }, + }, + xsrf: { + disableProtection: { type: 'boolean' }, + whitelistConfigured: { type: 'boolean' }, + }, + requestId: { + allowFromAnyIp: { type: 'boolean' }, + ipAllowlistConfigured: { type: 'boolean' }, + }, + ssl: { + certificateAuthoritiesConfigured: { type: 'boolean' }, + certificateConfigured: { type: 'boolean' }, + cipherSuites: { type: 'array', items: { type: 'keyword' } }, + keyConfigured: { type: 'boolean' }, + keystoreConfigured: { type: 'boolean' }, + truststoreConfigured: { type: 'boolean' }, + redirectHttpFromPortConfigured: { type: 'boolean' }, + supportedProtocols: { type: 'array', items: { type: 'keyword' } }, + clientAuthentication: { type: 'keyword' }, + }, + }, + + logging: { + appendersTypesUsed: { type: 'array', items: { type: 'keyword' } }, + loggersConfiguredCount: { type: 'long' }, + }, + + savedObjects: { + maxImportPayloadBytes: { type: 'long' }, + maxImportExportSizeBytes: { type: 'long' }, + }, + }, + environment: { + memory: { + heapSizeLimit: { type: 'long' }, + heapTotalBytes: { type: 'long' }, + heapUsedBytes: { type: 'long' }, + }, + }, + services: { + savedObjects: { + indices: { + type: 'array', + items: { + docsCount: { type: 'long' }, + docsDeleted: { type: 'long' }, + alias: { type: 'text' }, + primaryStoreSizeBytes: { type: 'long' }, + storeSizeBytes: { type: 'long' }, + }, + }, + }, + }, + }, + fetch() { + return getCoreUsageDataService().getCoreUsageData(); + }, + }); +} + +export function registerCoreUsageCollector( + usageCollection: UsageCollectionSetup, + getCoreUsageDataService: () => CoreUsageDataStart +) { + usageCollection.registerCollector( + getCoreUsageCollector(usageCollection, getCoreUsageDataService) + ); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts new file mode 100644 index 0000000000000..b712e9ebbce48 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + CollectorOptions, + createUsageCollectionSetupMock, +} from '../../../../usage_collection/server/usage_collection.mock'; + +import { registerCoreUsageCollector } from '.'; +import { coreUsageDataServiceMock } from '../../../../../core/server/mocks'; +import { CoreUsageData } from 'src/core/server/'; + +describe('telemetry_core', () => { + let collector: CollectorOptions; + + const usageCollectionMock = createUsageCollectionSetupMock(); + usageCollectionMock.makeUsageCollector.mockImplementation((config) => { + collector = config; + return createUsageCollectionSetupMock().makeUsageCollector(config); + }); + + const callCluster = jest.fn().mockImplementation(() => ({})); + const coreUsageDataStart = coreUsageDataServiceMock.createStartContract(); + const getCoreUsageDataReturnValue = (Symbol('core telemetry') as any) as CoreUsageData; + coreUsageDataStart.getCoreUsageData.mockResolvedValue(getCoreUsageDataReturnValue); + + beforeAll(() => registerCoreUsageCollector(usageCollectionMock, () => coreUsageDataStart)); + + test('registered collector is set', () => { + expect(collector).not.toBeUndefined(); + expect(collector.type).toBe('core'); + }); + + test('fetch', async () => { + expect(await collector.fetch(callCluster)).toEqual(getCoreUsageDataReturnValue); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/index.ts b/src/plugins/kibana_usage_collection/server/collectors/core/index.ts new file mode 100644 index 0000000000000..79a4b83b41355 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/core/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { registerCoreUsageCollector } from './core_usage_collector'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/index.ts b/src/plugins/kibana_usage_collection/server/collectors/index.ts index 1f9fe130fa45d..2408dc84c2e56 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/index.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/index.ts @@ -23,3 +23,4 @@ export { registerApplicationUsageCollector } from './application_usage'; export { registerKibanaUsageCollector } from './kibana'; export { registerOpsStatsCollector } from './ops_stats'; export { registerCspCollector } from './csp'; +export { registerCoreUsageCollector } from './core'; diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index 260acd19ab516..198fdbb7a8703 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -31,6 +31,7 @@ import { SavedObjectsServiceSetup, OpsMetrics, Logger, + CoreUsageDataStart, } from '../../../core/server'; import { registerApplicationUsageCollector, @@ -39,6 +40,7 @@ import { registerOpsStatsCollector, registerUiMetricUsageCollector, registerCspCollector, + registerCoreUsageCollector, } from './collectors'; interface KibanaUsageCollectionPluginsDepsSetup { @@ -53,6 +55,7 @@ export class KibanaUsageCollectionPlugin implements Plugin { private savedObjectsClient?: ISavedObjectsRepository; private uiSettingsClient?: IUiSettingsClient; private metric$: Subject; + private coreUsageData?: CoreUsageDataStart; constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); @@ -72,6 +75,7 @@ export class KibanaUsageCollectionPlugin implements Plugin { const savedObjectsClient = new SavedObjectsClient(this.savedObjectsClient); this.uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient); core.metrics.getOpsMetrics$().subscribe(this.metric$); + this.coreUsageData = core.coreUsageData; } public stop() { @@ -86,6 +90,7 @@ export class KibanaUsageCollectionPlugin implements Plugin { ) { const getSavedObjectsClient = () => this.savedObjectsClient; const getUiSettingsClient = () => this.uiSettingsClient; + const getCoreUsageDataService = () => this.coreUsageData!; registerOpsStatsCollector(usageCollection, metric$); registerKibanaUsageCollector(usageCollection, this.legacyConfig$); @@ -98,5 +103,6 @@ export class KibanaUsageCollectionPlugin implements Plugin { getSavedObjectsClient ); registerCspCollector(usageCollection, coreSetup.http); + registerCoreUsageCollector(usageCollection, getCoreUsageDataService); } }