From 8fc67c064af3f9e3f13329f9302ba8b50e745ffc Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Thu, 19 Jul 2018 12:03:57 -0700 Subject: [PATCH] Stats API: implement the "kibana status" spec from the Monitoring data model for stats (#20577) (#20956) * [Stats API] Set API field names per spec * fix jest tests * fix api integration test * trash the original metrics collector - constantly accumulating stats over time does not align with the existing behavior, which is to reset the stats to 0 whenever they are pulled * move some logic out of the collector types combiner into inline - change the signature of sourceKibana * Make a new stats collector for the API - to not clear the data when pulling via the api - fetching is a read-only thing * isolate data transforms for api data and upload data * no static methods * remove external in bytes * remove the _stats prefix for kibana and reporting * update jest test snapshot * fix collector_types_combiner test * fix usage api * add test suite todo comment * reduce some loc change * roll back mysterious change * reduce some more loc change * comment correction * reduce more loc change * whitespace * comment question * fix cluster_uuid * fix stats integration test * fix bulk uploader test, combineTypes is no longer external * very important comments about the current nature of stats represented and long-term goals * add stats api tests with/without authentication * fix more fields to match data model * fix more tests * fix jest test * remove TODO * remove sockets * use snake_case for api field names * restore accidental removal + copy/paste error * sourceKibana -> getKibanaInfoForStats * skip usage test on legacy endpoint * fix api tests * more comment * stop putting a field in that used to be omitted * fix the internal type to ID the usage data for bulk uploader * correct the kibana usage type value, which is shown as-is in the API * more fixes for the constants identifying collector types + test against duplicates * add a comment on a hack, and a whitespace fix --- src/server/kbn_server.js | 4 +- .../collectors/get_ops_stats_collector.js | 49 ++++ .../index.js | 2 +- src/server/status/constants.js | 20 ++ src/server/status/index.js | 17 +- .../__mocks__/_fs_stubs.js | 0 .../{metrics_collector => lib}/cgroup.js | 0 .../{metrics_collector => lib}/cgroup.test.js | 0 .../status/lib/get_kibana_info_for_stats.js | 46 +++ src/server/status/lib/index.js | 20 ++ .../{metrics_collector => lib}/metrics.js | 52 ++-- .../metrics.test.js | 61 ++-- .../metrics_collector.test.js.snap | 269 ------------------ .../metrics_collector/metrics_collector.js | 113 -------- .../metrics_collector.test.js | 164 ----------- .../metrics_collector/sum_accumulate.test.js | 101 ------- .../status/routes/api/register_stats.js | 53 ++-- src/server/usage/classes/collector_set.js | 48 +++- test/api_integration/apis/stats/stats.js | 142 ++++----- test/api_integration/apis/status/status.js | 11 +- x-pack/plugins/monitoring/common/constants.js | 3 +- x-pack/plugins/monitoring/init.js | 7 +- .../__tests__/bulk_uploader.js | 17 +- ...ulk_uploader.combine_stats_legacy.test.js} | 80 ++---- .../server/kibana_monitoring/bulk_uploader.js | 89 +++++- .../collectors/get_ops_stats_collector.js | 14 +- .../collectors/get_settings_collector.js | 8 +- .../ops_buffer/__tests__/event_roller.js | 4 - .../collectors/ops_buffer/ops_buffer.js | 4 - .../server/kibana_monitoring/init.js | 7 +- .../lib/get_collector_types_combiner.js | 89 ------ ...kibana.js => get_kibana_info_for_stats.js} | 7 +- .../server/kibana_monitoring/lib/index.js | 2 +- x-pack/plugins/reporting/common/constants.js | 2 +- .../server/routes/api/v1/xpack_usage.js | 6 +- x-pack/test/api_integration/apis/index.js | 1 + .../test/api_integration/apis/kibana/index.js | 11 + .../apis/kibana/stats/index.js | 11 + .../apis/kibana/stats/stats.js | 65 +++++ x-pack/test/reporting/api/usage.js | 6 +- .../test/reporting/services/reporting_api.js | 12 +- 41 files changed, 590 insertions(+), 1027 deletions(-) create mode 100644 src/server/status/collectors/get_ops_stats_collector.js rename src/server/status/{metrics_collector => collectors}/index.js (92%) create mode 100644 src/server/status/constants.js rename src/server/status/{metrics_collector => lib}/__mocks__/_fs_stubs.js (100%) rename src/server/status/{metrics_collector => lib}/cgroup.js (100%) rename src/server/status/{metrics_collector => lib}/cgroup.test.js (100%) create mode 100644 src/server/status/lib/get_kibana_info_for_stats.js create mode 100644 src/server/status/lib/index.js rename src/server/status/{metrics_collector => lib}/metrics.js (76%) rename src/server/status/{metrics_collector => lib}/metrics.test.js (84%) delete mode 100644 src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap delete mode 100644 src/server/status/metrics_collector/metrics_collector.js delete mode 100644 src/server/status/metrics_collector/metrics_collector.test.js delete mode 100644 src/server/status/metrics_collector/sum_accumulate.test.js rename x-pack/plugins/monitoring/server/kibana_monitoring/{lib/__tests__/get_collector_types_combiner.js => bulk_uploader.combine_stats_legacy.test.js} (70%) delete mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js rename x-pack/plugins/monitoring/server/kibana_monitoring/lib/{source_kibana.js => get_kibana_info_for_stats.js} (88%) create mode 100644 x-pack/test/api_integration/apis/kibana/index.js create mode 100644 x-pack/test/api_integration/apis/kibana/stats/index.js create mode 100644 x-pack/test/api_integration/apis/kibana/stats/stats.js diff --git a/src/server/kbn_server.js b/src/server/kbn_server.js index db4ddff754eab..dae90d64240c1 100644 --- a/src/server/kbn_server.js +++ b/src/server/kbn_server.js @@ -26,8 +26,8 @@ import configSetupMixin from './config/setup'; import httpMixin from './http'; import { loggingMixin } from './logging'; import warningsMixin from './warnings'; -import { statusMixin } from './status'; import { usageMixin } from './usage'; +import { statusMixin } from './status'; import pidMixin from './pid'; import { configDeprecationWarningsMixin } from './config/deprecation_warnings'; import configCompleteMixin from './config/complete'; @@ -68,8 +68,8 @@ export default class KbnServer { loggingMixin, configDeprecationWarningsMixin, warningsMixin, - statusMixin, usageMixin, + statusMixin, // writes pid file pidMixin, diff --git a/src/server/status/collectors/get_ops_stats_collector.js b/src/server/status/collectors/get_ops_stats_collector.js new file mode 100644 index 0000000000000..ba4a3d92699ae --- /dev/null +++ b/src/server/status/collectors/get_ops_stats_collector.js @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import { KIBANA_STATS_TYPE } from '../constants'; +import { getKibanaInfoForStats } from '../lib'; + +/* + * Initialize a collector for Kibana Ops Stats + * + * NOTE this collector's fetch method returns the latest stats from the + * Hapi/Good/Even-Better ops event listener. Therefore, the stats reset + * every 5 seconds (the default value of the ops.interval configuration + * setting). That makes it geared for providing the latest "real-time" + * stats. In the long-term, fetch should return stats that constantly + * accumulate over the server's uptime for better machine readability. + * Since the data is captured, timestamped and stored, the historical + * data can provide "real-time" stats by calculating a derivative of + * the metrics. + * See PR comment in https://github.com/elastic/kibana/pull/20577/files#r202416647 + */ +export function getOpsStatsCollector(server, kbnServer) { + const { collectorSet } = server.usage; + return collectorSet.makeStatsCollector({ + type: KIBANA_STATS_TYPE, + fetch: () => { + return { + kibana: getKibanaInfoForStats(server, kbnServer), + ...kbnServer.metrics // latest metrics captured from the ops event listener in src/server/status/index + }; + } + }); +} diff --git a/src/server/status/metrics_collector/index.js b/src/server/status/collectors/index.js similarity index 92% rename from src/server/status/metrics_collector/index.js rename to src/server/status/collectors/index.js index ebdecd61942ef..4310dff7359ef 100644 --- a/src/server/status/metrics_collector/index.js +++ b/src/server/status/collectors/index.js @@ -17,4 +17,4 @@ * under the License. */ -export { MetricsCollector } from './metrics_collector'; +export { getOpsStatsCollector } from './get_ops_stats_collector'; diff --git a/src/server/status/constants.js b/src/server/status/constants.js new file mode 100644 index 0000000000000..d16e46008acc8 --- /dev/null +++ b/src/server/status/constants.js @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const KIBANA_STATS_TYPE = 'kibana_stats'; // kibana stats per 5s intervals diff --git a/src/server/status/index.js b/src/server/status/index.js index 4367353f70a6f..f05a14e3367af 100644 --- a/src/server/status/index.js +++ b/src/server/status/index.js @@ -18,30 +18,29 @@ */ import ServerStatus from './server_status'; -import { MetricsCollector } from './metrics_collector'; -import { Metrics } from './metrics_collector/metrics'; +import { Metrics } from './lib/metrics'; import { registerStatusPage, registerStatusApi, registerStatsApi } from './routes'; +import { getOpsStatsCollector } from './collectors'; export function statusMixin(kbnServer, server, config) { - const collector = new MetricsCollector(server, config); kbnServer.status = new ServerStatus(kbnServer.server); + const statsCollector = getOpsStatsCollector(server, kbnServer); + const { collectorSet } = server.usage; + collectorSet.register(statsCollector); + const { ['even-better']: evenBetter } = server.plugins; if (evenBetter) { const metrics = new Metrics(config, server); evenBetter.monitor.on('ops', event => { - // for status API (to deprecate in next major) - metrics.capture(event).then(data => { kbnServer.metrics = data; }); - - // for metrics API (replacement API) - collector.collect(event); // collect() is async, but here we aren't depending on the return value + metrics.capture(event).then(data => { kbnServer.metrics = data; }); // captures (performs transforms on) the latest event data and stashes the metrics for status/stats API payload }); } // init routes registerStatusPage(kbnServer, server, config); registerStatusApi(kbnServer, server, config); - registerStatsApi(kbnServer, server, config, collector); + registerStatsApi(kbnServer, server, config); } diff --git a/src/server/status/metrics_collector/__mocks__/_fs_stubs.js b/src/server/status/lib/__mocks__/_fs_stubs.js similarity index 100% rename from src/server/status/metrics_collector/__mocks__/_fs_stubs.js rename to src/server/status/lib/__mocks__/_fs_stubs.js diff --git a/src/server/status/metrics_collector/cgroup.js b/src/server/status/lib/cgroup.js similarity index 100% rename from src/server/status/metrics_collector/cgroup.js rename to src/server/status/lib/cgroup.js diff --git a/src/server/status/metrics_collector/cgroup.test.js b/src/server/status/lib/cgroup.test.js similarity index 100% rename from src/server/status/metrics_collector/cgroup.test.js rename to src/server/status/lib/cgroup.test.js diff --git a/src/server/status/lib/get_kibana_info_for_stats.js b/src/server/status/lib/get_kibana_info_for_stats.js new file mode 100644 index 0000000000000..bac800c03b2b1 --- /dev/null +++ b/src/server/status/lib/get_kibana_info_for_stats.js @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { get } from 'lodash'; + +const snapshotRegex = /-snapshot/i; + +/** + * This provides a meta data attribute along with Kibana stats. + * + * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core + * @param {Object} config Server config + * @param {String} host Kibana host + * @return {Object} The object containing a "kibana" field and source instance details. + */ +export function getKibanaInfoForStats(server, kbnServer) { + const config = server.config(); + const status = kbnServer.status.toJSON(); + + return { + uuid: config.get('server.uuid'), + name: config.get('server.name'), + index: config.get('kibana.index'), + host: config.get('server.host'), + transport_address: `${config.get('server.host')}:${config.get('server.port')}`, + version: kbnServer.version.replace(snapshotRegex, ''), + snapshot: snapshotRegex.test(kbnServer.version), + status: get(status, 'overall.state') + }; +} diff --git a/src/server/status/lib/index.js b/src/server/status/lib/index.js new file mode 100644 index 0000000000000..93db8b2d22561 --- /dev/null +++ b/src/server/status/lib/index.js @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { getKibanaInfoForStats } from './get_kibana_info_for_stats'; diff --git a/src/server/status/metrics_collector/metrics.js b/src/server/status/lib/metrics.js similarity index 76% rename from src/server/status/metrics_collector/metrics.js rename to src/server/status/lib/metrics.js index f6499d44b32ee..1a7bcce65dfd7 100644 --- a/src/server/status/metrics_collector/metrics.js +++ b/src/server/status/lib/metrics.js @@ -18,6 +18,7 @@ */ import os from 'os'; +import v8 from 'v8'; import { get, isObject, merge } from 'lodash'; import { keysToSnakeCaseShallow } from '../../../utils/case_conversion'; import { getAllStats as cGroupStats } from './cgroup'; @@ -32,19 +33,17 @@ export class Metrics { static getStubMetrics() { return { process: { - mem: {} + memory: { + heap: {} + } }, os: { cpu: {}, - mem: {} + memory: {} }, response_times: {}, requests: { status_codes: {} - }, - sockets: { - http: {}, - https: {} } }; } @@ -56,53 +55,52 @@ export class Metrics { const metrics = { last_updated: timestamp, - collection_interval_in_millis: this.config.get('ops.interval'), - uptime_in_millis: event.process.uptime_ms, // TODO: deprecate this field, data should only have process.uptime_ms + collection_interval_in_millis: this.config.get('ops.interval') }; return merge(metrics, event, cgroup); } captureEvent(hapiEvent) { + const heapStats = v8.getHeapStatistics(); const port = this.config.get('server.port'); - const avgInMillis = get(hapiEvent, ['responseTimes', port, 'avg']); // sadly, it's possible for this to be NaN const maxInMillis = get(hapiEvent, ['responseTimes', port, 'max']); return { process: { - mem: { - // https://nodejs.org/docs/latest-v8.x/api/process.html#process_process_memoryusage - heap_max_in_bytes: get(hapiEvent, 'psmem.heapTotal'), - heap_used_in_bytes: get(hapiEvent, 'psmem.heapUsed'), + memory: { + heap: { + // https://nodejs.org/docs/latest-v8.x/api/process.html#process_process_memoryusage + total_in_bytes: get(hapiEvent, 'psmem.heapTotal'), + used_in_bytes: get(hapiEvent, 'psmem.heapUsed'), + size_limit: heapStats.heap_size_limit + }, resident_set_size_in_bytes: get(hapiEvent, 'psmem.rss'), - external_in_bytes: get(hapiEvent, 'psmem.external') }, + event_loop_delay: get(hapiEvent, 'psdelay'), pid: process.pid, - uptime_ms: process.uptime() * 1000 + uptime_in_millis: process.uptime() * 1000 }, os: { - cpu: { - load_average: { - '1m': get(hapiEvent, 'osload.0'), - '5m': get(hapiEvent, 'osload.1'), - '15m': get(hapiEvent, 'osload.2') - } + load: { + '1m': get(hapiEvent, 'osload.0'), + '5m': get(hapiEvent, 'osload.1'), + '15m': get(hapiEvent, 'osload.2') }, - mem: { + memory: { + total_in_bytes: os.totalmem(), free_in_bytes: os.freemem(), - total_in_bytes: os.totalmem() - } + used_in_bytes: get(hapiEvent, 'osmem.total') - get(hapiEvent, 'osmem.free') + }, + uptime_in_millis: os.uptime() * 1000 }, response_times: { - // TODO: rename to use `_ms` suffix per beats naming conventions avg_in_millis: isNaN(avgInMillis) ? undefined : avgInMillis, // convert NaN to undefined max_in_millis: maxInMillis }, requests: keysToSnakeCaseShallow(get(hapiEvent, ['requests', port])), concurrent_connections: get(hapiEvent, ['concurrents', port]), - sockets: get(hapiEvent, 'sockets'), - event_loop_delay: get(hapiEvent, 'psdelay') }; } diff --git a/src/server/status/metrics_collector/metrics.test.js b/src/server/status/lib/metrics.test.js similarity index 84% rename from src/server/status/metrics_collector/metrics.test.js rename to src/server/status/lib/metrics.test.js index 24459c28e981d..c0afa65749548 100644 --- a/src/server/status/metrics_collector/metrics.test.js +++ b/src/server/status/lib/metrics.test.js @@ -23,10 +23,13 @@ jest.mock('fs', () => ({ jest.mock('os', () => ({ freemem: jest.fn(), - totalmem: jest.fn() + totalmem: jest.fn(), + uptime: jest.fn() })); -jest.mock('process'); +jest.mock('process', () => ({ + uptime: jest.fn() +})); import fs from 'fs'; import os from 'os'; @@ -69,10 +72,9 @@ describe('Metrics', function () { sinon.stub(Date.prototype, 'toISOString').returns('2017-04-14T18:35:41.534Z'); const capturedMetrics = await metrics.capture(); - expect(capturedMetrics).toEqual({ + expect(capturedMetrics).toMatchObject({ last_updated: '2017-04-14T18:35:41.534Z', collection_interval_in_millis: 5000, - uptime_in_millis: 1980, a: [ { b: 2, c: 3 }, { d: 4, e: 5 } ], process: { uptime_ms: 1980 } }); }); @@ -80,6 +82,7 @@ describe('Metrics', function () { describe('captureEvent', () => { it('parses the hapi event', () => { + sinon.stub(os, 'uptime').returns(12000); sinon.stub(process, 'uptime').returns(5000); os.freemem.mockImplementation(() => 12); @@ -92,10 +95,6 @@ describe('Metrics', function () { const hapiEvent = { 'requests': { '5603': { 'total': 22, 'disconnects': 0, 'statusCodes': { '200': 22 } } }, 'responseTimes': { '5603': { 'avg': 1.8636363636363635, 'max': 4 } }, - 'sockets': { - 'http': { 'total': 0 }, - 'https': { 'total': 0 } - }, 'osload': [2.20751953125, 2.02294921875, 1.89794921875], 'osmem': { 'total': 17179869184, 'free': 102318080 }, 'osup': 1008991, @@ -106,31 +105,29 @@ describe('Metrics', function () { 'host': 'blahblah.local' }; - expect(metrics.captureEvent(hapiEvent)).toEqual({ + expect(metrics.captureEvent(hapiEvent)).toMatchObject({ 'concurrent_connections': 0, - 'event_loop_delay': 1.6091690063476562, 'os': { - 'cpu': { - 'load_average': { - '15m': 1.89794921875, - '1m': 2.20751953125, - '5m': 2.02294921875 - } + 'load': { + '15m': 1.89794921875, + '1m': 2.20751953125, + '5m': 2.02294921875 }, - 'mem': { + 'memory': { 'free_in_bytes': 12, 'total_in_bytes': 24, }, + 'uptime_in_millis': 12000000, }, 'process': { - 'mem': { - 'external_in_bytes': 1779619, - 'heap_max_in_bytes': 168194048, - 'heap_used_in_bytes': 130553400, + 'memory': { + 'heap': { + 'total_in_bytes': 168194048, + 'used_in_bytes': 130553400, + }, 'resident_set_size_in_bytes': 193716224, }, - 'pid': 8675309, - 'uptime_ms': 5000000 + 'pid': 8675309 }, 'requests': { 'disconnects': 0, @@ -143,14 +140,6 @@ describe('Metrics', function () { 'avg_in_millis': 1.8636363636363635, 'max_in_millis': 4 }, - 'sockets': { - 'http': { - 'total': 0 - }, - 'https': { - 'total': 0 - } - } }); }); @@ -163,11 +152,11 @@ describe('Metrics', function () { host: 'blahblah.local', }; - expect(metrics.captureEvent(hapiEvent)).toEqual({ - process: { mem: {}, pid: 8675309, uptime_ms: 5000000 }, + expect(metrics.captureEvent(hapiEvent)).toMatchObject({ + process: { memory: { heap: {} }, pid: 8675309, uptime_in_millis: 5000000 }, os: { - cpu: { load_average: {} }, - mem: { free_in_bytes: 12, total_in_bytes: 24 }, + load: {}, + memory: { free_in_bytes: 12, total_in_bytes: 24 }, }, response_times: { max_in_millis: 4 }, requests: { total: 22, disconnects: 0, status_codes: { '200': 22 } }, @@ -194,7 +183,7 @@ describe('Metrics', function () { const capturedMetrics = await metrics.captureCGroups(); - expect(capturedMetrics).toEqual({ + expect(capturedMetrics).toMatchObject({ os: { cgroup: { cpuacct: { diff --git a/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap b/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap deleted file mode 100644 index 06b9b275f7200..0000000000000 --- a/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap +++ /dev/null @@ -1,269 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`Metrics Collector collection should accumulate counter metrics 1`] = ` -Object { - "collection_interval_in_millis": "test-123", - "concurrent_connections": 0, - "event_loop_delay": 0.3764979839324951, - "last_updated": "2018-04-19T21:50:54.366Z", - "name": "test-123", - "os": Object { - "cpu": Object { - "load_average": Object { - "15m": 1.81201171875, - "1m": 1.97119140625, - "5m": 1.90283203125, - }, - }, - "mem": Object { - "free_in_bytes": 12, - "total_in_bytes": 24, - }, - }, - "process": Object { - "mem": Object { - "external_in_bytes": 25028, - "heap_max_in_bytes": 15548416, - "heap_used_in_bytes": 12996392, - "resident_set_size_in_bytes": 36085760, - }, - "pid": 7777, - "uptime_ms": 6666000, - }, - "requests": Object { - "disconnects": 0, - "status_codes": Object { - "200": 8, - }, - "total": 8, - }, - "response_times": Object { - "avg_in_millis": 19, - "max_in_millis": 19, - }, - "sockets": Object { - "http": Object { - "total": 0, - }, - "https": Object { - "total": 0, - }, - }, - "uptime_in_millis": 6666000, - "uuid": "test-123", - "version": Object { - "build_hash": "test-123", - "build_number": "test-123", - "build_snapshot": false, - "number": "test-123", - }, -} -`; - -exports[`Metrics Collector collection should accumulate counter metrics 2`] = ` -Object { - "collection_interval_in_millis": "test-123", - "concurrent_connections": 0, - "event_loop_delay": 0.7529959678649902, - "last_updated": "2018-04-19T21:50:54.366Z", - "name": "test-123", - "os": Object { - "cpu": Object { - "load_average": Object { - "15m": 1.81201171875, - "1m": 1.97119140625, - "5m": 1.90283203125, - }, - }, - "mem": Object { - "free_in_bytes": 12, - "total_in_bytes": 24, - }, - }, - "process": Object { - "mem": Object { - "external_in_bytes": 25028, - "heap_max_in_bytes": 15548416, - "heap_used_in_bytes": 12996392, - "resident_set_size_in_bytes": 36085760, - }, - "pid": 7777, - "uptime_ms": 6666000, - }, - "requests": Object { - "disconnects": 0, - "status_codes": Object { - "200": 16, - }, - "total": 16, - }, - "response_times": Object { - "avg_in_millis": 38, - "max_in_millis": 38, - }, - "sockets": Object { - "http": Object { - "total": 0, - }, - "https": Object { - "total": 0, - }, - }, - "uptime_in_millis": 6666000, - "uuid": "test-123", - "version": Object { - "build_hash": "test-123", - "build_number": "test-123", - "build_snapshot": false, - "number": "test-123", - }, -} -`; - -exports[`Metrics Collector collection should accumulate counter metrics 3`] = ` -Object { - "collection_interval_in_millis": "test-123", - "concurrent_connections": 0, - "event_loop_delay": 1.1294939517974854, - "last_updated": "2018-04-19T21:50:54.366Z", - "name": "test-123", - "os": Object { - "cpu": Object { - "load_average": Object { - "15m": 1.81201171875, - "1m": 1.97119140625, - "5m": 1.90283203125, - }, - }, - "mem": Object { - "free_in_bytes": 12, - "total_in_bytes": 24, - }, - }, - "process": Object { - "mem": Object { - "external_in_bytes": 25028, - "heap_max_in_bytes": 15548416, - "heap_used_in_bytes": 12996392, - "resident_set_size_in_bytes": 36085760, - }, - "pid": 7777, - "uptime_ms": 6666000, - }, - "requests": Object { - "disconnects": 0, - "status_codes": Object { - "200": 24, - }, - "total": 24, - }, - "response_times": Object { - "avg_in_millis": 57, - "max_in_millis": 57, - }, - "sockets": Object { - "http": Object { - "total": 0, - }, - "https": Object { - "total": 0, - }, - }, - "uptime_in_millis": 6666000, - "uuid": "test-123", - "version": Object { - "build_hash": "test-123", - "build_number": "test-123", - "build_snapshot": false, - "number": "test-123", - }, -} -`; - -exports[`Metrics Collector collection should update stats with new data 1`] = ` -Object { - "collection_interval_in_millis": "test-123", - "concurrent_connections": 0, - "event_loop_delay": 0.33843398094177246, - "last_updated": "2018-04-19T21:50:54.366Z", - "name": "test-123", - "os": Object { - "cpu": Object { - "load_average": Object { - "15m": 1.8154296875, - "1m": 1.68017578125, - "5m": 1.7685546875, - }, - }, - "mem": Object { - "free_in_bytes": 12, - "total_in_bytes": 24, - }, - }, - "process": Object { - "mem": Object { - "external_in_bytes": 25028, - "heap_max_in_bytes": 15548416, - "heap_used_in_bytes": 12911128, - "resident_set_size_in_bytes": 35307520, - }, - "pid": 7777, - "uptime_ms": 6666000, - }, - "requests": Object { - "disconnects": 0, - "status_codes": Object { - "200": 4, - }, - "total": 4, - }, - "response_times": Object { - "avg_in_millis": 13, - "max_in_millis": 13, - }, - "sockets": Object { - "http": Object { - "total": 0, - }, - "https": Object { - "total": 0, - }, - }, - "uptime_in_millis": 6666000, - "uuid": "test-123", - "version": Object { - "build_hash": "test-123", - "build_number": "test-123", - "build_snapshot": false, - "number": "test-123", - }, -} -`; - -exports[`Metrics Collector initialize should return stub metrics 1`] = ` -Object { - "name": "test-123", - "os": Object { - "cpu": Object {}, - "mem": Object {}, - }, - "process": Object { - "mem": Object {}, - }, - "requests": Object { - "status_codes": Object {}, - }, - "response_times": Object {}, - "sockets": Object { - "http": Object {}, - "https": Object {}, - }, - "uuid": "test-123", - "version": Object { - "build_hash": "test-123", - "build_number": "test-123", - "build_snapshot": false, - "number": "test-123", - }, -} -`; diff --git a/src/server/status/metrics_collector/metrics_collector.js b/src/server/status/metrics_collector/metrics_collector.js deleted file mode 100644 index 1d3ee4fb63b48..0000000000000 --- a/src/server/status/metrics_collector/metrics_collector.js +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { Metrics } from './metrics'; - -const matchSnapshot = /-SNAPSHOT$/; - -/* - * Persist operational data for machine reading - * sets the latest gauge values - * sums the latest accumulative values - */ -export class MetricsCollector { - constructor(server, config) { - - // NOTE we need access to config every time this is used because uuid is managed by the kibana core_plugin, which is initialized AFTER kbn_server - this._getBaseStats = () => ({ - name: config.get('server.name'), - uuid: config.get('server.uuid'), - version: { - number: config.get('pkg.version').replace(matchSnapshot, ''), - build_hash: config.get('pkg.buildSha'), - build_number: config.get('pkg.buildNum'), - build_snapshot: matchSnapshot.test(config.get('pkg.version')) - } - }); - - this._stats = Metrics.getStubMetrics(); - this._metrics = new Metrics(config, server); // TODO: deprecate status API that uses Metrics class, move it this module, fix the order of its constructor params - } - - /* - * Accumulate metrics by summing values in an accumulutor object with the next values - * - * @param {String} property The property of the objects to roll up - * @param {Object} accum The accumulator object - * @param {Object} next The object containing next values - */ - static sumAccumulate(property, accum, next) { - const accumValue = accum[property]; - const nextValue = next[property]; - - if (nextValue === null || nextValue === undefined) { - return; // do not accumulate null/undefined since it can't be part of a sum - } else if (nextValue.constructor === Object) { // nested structure, object - const newProps = {}; - for (const innerKey in nextValue) { - if (nextValue.hasOwnProperty(innerKey)) { - const tempAccumValue = accumValue || {}; - newProps[innerKey] = MetricsCollector.sumAccumulate(innerKey, tempAccumValue, nextValue); - } - } - return { // merge the newly summed nested values - ...accumValue, - ...newProps, - }; - } else if (nextValue.constructor === Number) { - // leaf value - if (nextValue || nextValue === 0) { - const tempAccumValue = accumValue || 0; // treat null / undefined as 0 - const tempNextValue = nextValue || 0; - return tempAccumValue + tempNextValue; // perform sum - } - } else { - return; // drop unknown type - } - } - - async collect(event) { - const capturedEvent = await this._metrics.capture(event); // wait for cgroup measurement - const { process, os, ...metrics } = capturedEvent; - - const stats = { - // gauge values - ...metrics, - process, - os, - - // accumulative counters - response_times: MetricsCollector.sumAccumulate('response_times', this._stats, metrics), - requests: MetricsCollector.sumAccumulate('requests', this._stats, metrics), - concurrent_connections: MetricsCollector.sumAccumulate('concurrent_connections', this._stats, metrics), - sockets: MetricsCollector.sumAccumulate('sockets', this._stats, metrics), - event_loop_delay: MetricsCollector.sumAccumulate('event_loop_delay', this._stats, metrics), - }; - - this._stats = stats; - return stats; - } - - getStats() { - return { - ...this._getBaseStats(), - ...this._stats - }; - } -} diff --git a/src/server/status/metrics_collector/metrics_collector.test.js b/src/server/status/metrics_collector/metrics_collector.test.js deleted file mode 100644 index e6107d6b3e3c0..0000000000000 --- a/src/server/status/metrics_collector/metrics_collector.test.js +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -jest.mock('os', () => ({ - freemem: jest.fn(), - totalmem: jest.fn() -})); - -const mockProcessUptime = jest.fn().mockImplementation(() => 6666); -jest.mock('process', () => ({ - uptime: mockProcessUptime -})); - -import os from 'os'; -import sinon from 'sinon'; -import { MetricsCollector } from './'; - -const mockServer = {}; -const mockConfig = { - get: sinon.stub(), -}; -mockConfig.get.returns('test-123'); -mockConfig.get.withArgs('server.port').returns(3000); - -describe('Metrics Collector', () => { - describe('initialize', () => { - it('should return stub metrics', () => { - const collector = new MetricsCollector(mockServer, mockConfig); - expect(collector.getStats()).toMatchSnapshot(); - }); - }); - - describe('collection', () => { - os.freemem.mockImplementation(() => 12); - os.totalmem.mockImplementation(() => 24); - - Object.defineProperty(process, 'pid', { value: 7777 }); - Object.defineProperty(process, 'uptime', { value: mockProcessUptime }); - - let sandbox; - let clock; - beforeAll(() => { - sandbox = sinon.createSandbox(); - clock = sandbox.useFakeTimers(1524174654366); - }); - - afterAll(() => { - clock.restore(); - sandbox.restore(); - }); - - it('should update stats with new data', async () => { - const collector = new MetricsCollector(mockServer, mockConfig); - - await collector.collect({ - requests: { - '3000': { total: 4, disconnects: 0, statusCodes: { '200': 4 } }, - }, - responseTimes: { '3000': { avg: 13, max: 13 } }, - sockets: { http: { total: 0 }, https: { total: 0 } }, - osload: [1.68017578125, 1.7685546875, 1.8154296875], - osmem: { total: 17179869184, free: 3984404480 }, - psmem: { - rss: 35307520, - heapTotal: 15548416, - heapUsed: 12911128, - external: 25028, - }, - concurrents: { '3000': 0 }, - osup: 965002, - psup: 29.466, - psdelay: 0.33843398094177246, - host: 'spicy.local', - }); - expect(collector.getStats()).toMatchSnapshot(); - }); - - it('should accumulate counter metrics', async () => { - const collector = new MetricsCollector(mockServer, mockConfig); - - await collector.collect({ - requests: { - '3000': { total: 8, disconnects: 0, statusCodes: { '200': 8 } }, - }, - responseTimes: { '3000': { avg: 19, max: 19 } }, - sockets: { http: { total: 0 }, https: { total: 0 } }, - osload: [1.97119140625, 1.90283203125, 1.81201171875], - osmem: { total: 17179869184, free: 3987533824 }, - psmem: { - rss: 36085760, - heapTotal: 15548416, - heapUsed: 12996392, - external: 25028, - }, - concurrents: { '3000': 0 }, - osup: 965606, - psup: 22.29, - psdelay: 0.3764979839324951, - host: 'spicy.local', - }); - expect(collector.getStats()).toMatchSnapshot(); - - await collector.collect({ - requests: { - '3000': { total: 8, disconnects: 0, statusCodes: { '200': 8 } }, - }, - responseTimes: { '3000': { avg: 19, max: 19 } }, - sockets: { http: { total: 0 }, https: { total: 0 } }, - osload: [1.97119140625, 1.90283203125, 1.81201171875], - osmem: { total: 17179869184, free: 3987533824 }, - psmem: { - rss: 36085760, - heapTotal: 15548416, - heapUsed: 12996392, - external: 25028, - }, - concurrents: { '3000': 0 }, - osup: 965606, - psup: 22.29, - psdelay: 0.3764979839324951, - host: 'spicy.local', - }); - expect(collector.getStats()).toMatchSnapshot(); - - await collector.collect({ - requests: { - '3000': { total: 8, disconnects: 0, statusCodes: { '200': 8 } }, - }, - responseTimes: { '3000': { avg: 19, max: 19 } }, - sockets: { http: { total: 0 }, https: { total: 0 } }, - osload: [1.97119140625, 1.90283203125, 1.81201171875], - osmem: { total: 17179869184, free: 3987533824 }, - psmem: { - rss: 36085760, - heapTotal: 15548416, - heapUsed: 12996392, - external: 25028, - }, - concurrents: { '3000': 0 }, - osup: 965606, - psup: 22.29, - psdelay: 0.3764979839324951, - host: 'spicy.local', - }); - expect(collector.getStats()).toMatchSnapshot(); - }); - }); -}); diff --git a/src/server/status/metrics_collector/sum_accumulate.test.js b/src/server/status/metrics_collector/sum_accumulate.test.js deleted file mode 100644 index 660a1d015a873..0000000000000 --- a/src/server/status/metrics_collector/sum_accumulate.test.js +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { MetricsCollector } from './'; - -const { sumAccumulate } = MetricsCollector; - -describe('Accumulate By Summing Metrics', function () { - it('should accumulate empty object with nothing as nothing', () => { - const accum = { blues: {} }; - const current = sumAccumulate('blues', accum, {}); - expect(current).toEqual(undefined); - }); - - it('should return data to merge with initial empty data', () => { - let accum = { blues: {} }; - const next = { blues: { total: 1 } }; - const accumulated = sumAccumulate('blues', accum, next); - accum = { ...accum, blues: accumulated }; - expect(accum).toEqual({ blues: { total: 1 } }); - }); - - it('should return data to merge with already accumulated data', () => { - let currentProp; - let accumulated; - - // initial - let accum = { - reds: 1, - oranges: { total: 2 }, - yellows: { total: 3 }, - greens: { total: 4 }, - blues: { dislikes: 2, likes: 3, total: 5 }, - indigos: { total: 6 }, - violets: { total: 7 }, - }; - - // first accumulation - existing nested object - currentProp = 'blues'; - accumulated = sumAccumulate(currentProp, accum, { - [currentProp]: { likes: 2, total: 2 }, - }); - accum = { ...accum, [currentProp]: accumulated }; - expect(accum).toEqual({ - reds: 1, - oranges: { total: 2 }, - yellows: { total: 3 }, - greens: { total: 4 }, - blues: { dislikes: 2, likes: 5, total: 7 }, - indigos: { total: 6 }, - violets: { total: 7 }, - }); - - // second accumulation - existing non-nested object - currentProp = 'reds'; - accumulated = sumAccumulate(currentProp, accum, { [currentProp]: 2 }); - accum = { ...accum, [currentProp]: accumulated }; - expect(accum).toEqual({ - reds: 3, - oranges: { total: 2 }, - yellows: { total: 3 }, - greens: { total: 4 }, - blues: { dislikes: 2, likes: 5, total: 7 }, - indigos: { total: 6 }, - violets: { total: 7 }, - }); - - // third accumulation - new nested object prop - currentProp = 'ultraviolets'; - accumulated = sumAccumulate(currentProp, accum, { - [currentProp]: { total: 1, likes: 1, dislikes: 0 }, - }); - accum = { ...accum, [currentProp]: accumulated }; - expect(accum).toEqual({ - reds: 3, - oranges: { total: 2 }, - yellows: { total: 3 }, - greens: { total: 4 }, - blues: { dislikes: 2, likes: 5, total: 7 }, - indigos: { total: 6 }, - violets: { total: 7 }, - ultraviolets: { dislikes: 0, likes: 1, total: 1 }, - }); - }); -}); diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js index 296fb532573c9..8eb69e96d94c3 100644 --- a/src/server/status/routes/api/register_stats.js +++ b/src/server/status/routes/api/register_stats.js @@ -18,19 +18,33 @@ */ import Joi from 'joi'; +import { boomify } from 'boom'; import { wrapAuthConfig } from '../../wrap_auth_config'; +import { KIBANA_STATS_TYPE } from '../../constants'; /* * API for Kibana meta info and accumulated operations stats - * Including ?extended in the query string fetches Elasticsearch cluster_uuid + * Including ?extended in the query string fetches Elasticsearch cluster_uuid and server.usage.collectorSet data * - Requests to set isExtended = true * GET /api/stats?extended=true * GET /api/stats?extended * - No value or 'false' is isExtended = false * - Any other value causes a statusCode 400 response (Bad Request) */ -export function registerStatsApi(kbnServer, server, config, collector) { +export function registerStatsApi(kbnServer, server, config) { const wrapAuth = wrapAuthConfig(config.get('status.allowAnonymous')); + const { collectorSet } = server.usage; + + const getClusterUuid = async callCluster => { + const { cluster_uuid: uuid } = await callCluster('info', { filterPath: 'cluster_uuid', }); + return uuid; + }; + + const getUsage = async callCluster => { + const usage = await collectorSet.bulkFetchUsage(callCluster); + return collectorSet.toObject(usage); + }; + server.route( wrapAuth({ method: 'GET', @@ -44,27 +58,34 @@ export function registerStatsApi(kbnServer, server, config, collector) { tags: ['api'], }, async handler(req, reply) { - const { extended } = req.query; - const isExtended = extended !== undefined && extended !== 'false'; + const isExtended = req.query.extended !== undefined && req.query.extended !== 'false'; - let clusterUuid; + let extended; if (isExtended) { + const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('admin'); + const callCluster = (...args) => callWithRequest(req, ...args); try { - const { callWithRequest, } = server.plugins.elasticsearch.getCluster('data'); - const { cluster_uuid: uuid } = await callWithRequest(req, 'info', { filterPath: 'cluster_uuid', }); - clusterUuid = uuid; - } catch (err) { - clusterUuid = undefined; // fallback from anonymous access or auth failure, redundant for explicitness + const [ usage, clusterUuid ] = await Promise.all([ + getUsage(callCluster), + getClusterUuid(callCluster), + ]); + extended = collectorSet.toApiFieldNames({ usage, clusterUuid }); + } catch (e) { + return reply(boomify(e)); } } - const stats = { - cluster_uuid: clusterUuid, // serialization makes an undefined get stripped out, as undefined isn't a JSON type - status: kbnServer.status.toJSON(), - ...collector.getStats(), - }; + /* kibana_stats gets singled out from the collector set as it is used + * for health-checking Kibana and fetch does not rely on fetching data + * from ES */ + const kibanaStatsCollector = collectorSet.getCollectorByType(KIBANA_STATS_TYPE); + let kibanaStats = await kibanaStatsCollector.fetch(); + kibanaStats = collectorSet.toApiFieldNames(kibanaStats); - reply(stats); + reply({ + ...kibanaStats, + ...extended, + }); }, }) ); diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js index 4ce2a88776326..91834565bfab3 100644 --- a/src/server/usage/classes/collector_set.js +++ b/src/server/usage/classes/collector_set.js @@ -17,6 +17,7 @@ * under the License. */ +import { snakeCase } from 'lodash'; import Promise from 'bluebird'; import { getCollectorLogger } from '../lib'; import { Collector } from './collector'; @@ -64,6 +65,10 @@ export class CollectorSet { } } + getCollectorByType(type) { + return this._collectors.find(c => c.type === type); + } + /* * Call a bunch of fetch methods and then do them in bulk * @param {Array} collectors - an array of collectors, default to all registered collectors @@ -89,18 +94,41 @@ export class CollectorSet { async bulkFetchUsage(callCluster) { const usageCollectors = this._collectors.filter(c => c instanceof UsageCollector); - const bulk = await this.bulkFetch(callCluster, usageCollectors); - - // summarize each type of stat - return bulk.reduce((accumulatedStats, currentStat) => { - /* Suffix removal is a temporary hack: some types have `_stats` suffix - * because of how monitoring bulk upload needed to combine types. It can - * be removed when bulk upload goes away - */ - const statType = currentStat.type.replace('_stats', ''); + return this.bulkFetch(callCluster, usageCollectors); + } + + // convert an array of fetched stats results into key/object + toObject(statsData) { + return statsData.reduce((accumulatedStats, { type, result }) => { return { ...accumulatedStats, - [statType]: currentStat.result, + [type]: result, + }; + }, {}); + } + + // rename fields to use api conventions + toApiFieldNames(apiData) { + const getValueOrRecurse = value => { + if (value == null || typeof value !== 'object') { + return value; + } else { + return this.toApiFieldNames(value); // recurse + } + }; + + return Object.keys(apiData).reduce((accum, currName) => { + const value = apiData[currName]; + + let newName = currName; + newName = snakeCase(newName); + newName = newName.replace(/^(1|5|15)_m/, '$1m'); // os.load.15m, os.load.5m, os.load.1m + newName = newName.replace('_in_bytes', '_bytes'); + newName = newName.replace('_in_millis', '_ms'); + + return { + ...accum, + [newName]: getValueOrRecurse(value), }; }, {}); } diff --git a/test/api_integration/apis/stats/stats.js b/test/api_integration/apis/stats/stats.js index 6e9bbb801482f..4f25913b640c7 100644 --- a/test/api_integration/apis/stats/stats.js +++ b/test/api_integration/apis/stats/stats.js @@ -20,85 +20,91 @@ import expect from 'expect.js'; const assertStatsAndMetrics = body => { - expect(body.status.overall.state).to.be('green'); - expect(body.status.statuses).to.be.an('array'); - const kibanaPlugin = body.status.statuses.find(s => { - return s.id.indexOf('plugin:kibana') === 0; - }); - expect(kibanaPlugin.state).to.be('green'); - - expect(body.name).to.be.a('string'); - expect(body.uuid).to.be.a('string'); - - expect(body.version.number).to.be.a('string'); - - expect(body.process.mem.external_in_bytes).to.be.an('number'); - expect(body.process.mem.heap_max_in_bytes).to.be.an('number'); - expect(body.process.mem.heap_used_in_bytes).to.be.an('number'); - expect(body.process.mem.resident_set_size_in_bytes).to.be.an('number'); - expect(body.process.pid).to.be.an('number'); - expect(body.process.uptime_ms).to.be.an('number'); - - expect(body.os.cpu.load_average['1m']).to.be.a('number'); - - expect(body.response_times.avg_in_millis).not.to.be(null); // ok if is undefined - expect(body.response_times.max_in_millis).not.to.be(null); // ok if is undefined + expect(body.kibana.name).to.be.a('string'); + expect(body.kibana.uuid).to.be.a('string'); + expect(body.kibana.host).to.be.a('string'); + expect(body.kibana.transport_address).to.be.a('string'); + expect(body.kibana.version).to.be.a('string'); + expect(body.kibana.snapshot).to.be.a('boolean'); + expect(body.kibana.status).to.be('green'); + + expect(body.process.memory.heap.total_bytes).to.be.a('number'); + expect(body.process.memory.heap.used_bytes).to.be.a('number'); + expect(body.process.memory.heap.size_limit).to.be.a('number'); + expect(body.process.memory.resident_set_size_bytes).to.be.a('number'); + expect(body.process.pid).to.be.a('number'); + expect(body.process.uptime_ms).to.be.a('number'); + expect(body.process.event_loop_delay).to.be.a('number'); + + expect(body.os.memory.free_bytes).to.be.a('number'); + expect(body.os.memory.total_bytes).to.be.a('number'); + expect(body.os.uptime_ms).to.be.a('number'); + + expect(body.os.load['1m']).to.be.a('number'); + expect(body.os.load['5m']).to.be.a('number'); + expect(body.os.load['15m']).to.be.a('number'); + + expect(body.response_times.avg_ms).not.to.be(null); // ok if is undefined + expect(body.response_times.max_ms).not.to.be(null); // ok if is undefined expect(body.requests.status_codes).to.be.an('object'); - - expect(body.sockets.http).to.be.an('object'); - expect(body.sockets.https).to.be.an('object'); + expect(body.requests.total).to.be.a('number'); + expect(body.requests.disconnects).to.be.a('number'); expect(body.concurrent_connections).to.be.a('number'); - - expect(body.event_loop_delay).to.be.an('number'); }; export default function ({ getService }) { const supertest = getService('supertest'); - describe('kibana stats api', () => { - it('should return the stats and metric fields without cluster_uuid when extended param is not present', () => { - return supertest - .get('/api/stats') - .expect('Content-Type', /json/) - .expect(200) - .then(({ body }) => { - expect(body.cluster_uuid).to.be(undefined); - assertStatsAndMetrics(body); - }); - }); - it('should return the stats and metric fields without cluster_uuid when extended param is given as false', () => { - return supertest - .get('/api/stats?extended=false') - .expect('Content-Type', /json/) - .expect(200) - .then(({ body }) => { - expect(body.cluster_uuid).to.be(undefined); - assertStatsAndMetrics(body); - }); - }); - - it('should return the stats and metric fields with cluster_uuid when extended param is present', () => { - return supertest - .get('/api/stats?extended') - .expect('Content-Type', /json/) - .expect(200) - .then(({ body }) => { - expect(body.cluster_uuid).to.be.a('string'); - assertStatsAndMetrics(body); - }); + describe('basic', () => { + it('should return the stats without cluster_uuid with no query string params', () => { + return supertest + .get('/api/stats') + .expect('Content-Type', /json/) + .expect(200) + .then(({ body }) => { + expect(body.cluster_uuid).to.be(undefined); + assertStatsAndMetrics(body); + }); + }); + it(`should return the stats without cluster_uuid with 'extended' query string param = false`, () => { + return supertest + .get('/api/stats?extended=false') + .expect('Content-Type', /json/) + .expect(200) + .then(({ body }) => { + expect(body.cluster_uuid).to.be(undefined); + assertStatsAndMetrics(body); + }); + }); }); - it('should return the stats and metric fields with cluster_uuid when extended param is given as true', () => { - return supertest - .get('/api/stats?extended=true') - .expect('Content-Type', /json/) - .expect(200) - .then(({ body }) => { - expect(body.cluster_uuid).to.be.a('string'); - assertStatsAndMetrics(body); - }); + // TODO load an es archive and verify the counts in saved object usage info + describe('extended', () => { + it(`should return the stats, cluster_uuid, and usage with 'extended' query string param present`, () => { + return supertest + .get('/api/stats?extended') + .expect('Content-Type', /json/) + .expect(200) + .then(({ body }) => { + expect(body.cluster_uuid).to.be.a('string'); + expect(body.usage).to.be.an('object'); // no usage collectors have been registered so usage is an empty object + assertStatsAndMetrics(body); + }); + }); + + it(`should return the stats, cluster_uuid, and usage with 'extended' query string param = true`, () => { + return supertest + .get('/api/stats?extended=true') + .expect('Content-Type', /json/) + .expect(200) + .then(({ body }) => { + expect(body.cluster_uuid).to.be.a('string'); + expect(body.usage).to.be.an('object'); + assertStatsAndMetrics(body); + }); + }); }); }); } diff --git a/test/api_integration/apis/status/status.js b/test/api_integration/apis/status/status.js index a64bdc8ebf862..969558d38f93b 100644 --- a/test/api_integration/apis/status/status.js +++ b/test/api_integration/apis/status/status.js @@ -46,12 +46,13 @@ export default function ({ getService }) { expect(body.metrics.collection_interval_in_millis).to.be.a('number'); - expect(body.metrics.process.mem.heap_max_in_bytes).to.be.a('number'); - expect(body.metrics.process.mem.heap_used_in_bytes).to.be.a('number'); + expect(body.metrics.process.memory.heap.total_in_bytes).to.be.a('number'); + expect(body.metrics.process.memory.heap.used_in_bytes).to.be.a('number'); + expect(body.metrics.process.memory.heap.size_limit).to.be.a('number'); - expect(body.metrics.os.cpu.load_average['1m']).to.be.a('number'); - expect(body.metrics.os.cpu.load_average['5m']).to.be.a('number'); - expect(body.metrics.os.cpu.load_average['15m']).to.be.a('number'); + expect(body.metrics.os.load['1m']).to.be.a('number'); + expect(body.metrics.os.load['5m']).to.be.a('number'); + expect(body.metrics.os.load['15m']).to.be.a('number'); expect(body.metrics.response_times.avg_in_millis).not.to.be(null); // ok if undefined expect(body.metrics.response_times.max_in_millis).not.to.be(null); // ok if undefined diff --git a/x-pack/plugins/monitoring/common/constants.js b/x-pack/plugins/monitoring/common/constants.js index ed33d85dd41fd..30b700c48ae0e 100644 --- a/x-pack/plugins/monitoring/common/constants.js +++ b/x-pack/plugins/monitoring/common/constants.js @@ -22,7 +22,7 @@ export const MONITORING_SYSTEM_API_VERSION = '6'; * The type name used within the Monitoring index to publish Kibana ops stats. * @type {string} */ -export const KIBANA_STATS_TYPE = 'kibana_stats'; +export const KIBANA_STATS_TYPE_MONITORING = 'kibana_stats_monitoring'; // similar to KIBANA_STATS_TYPE but rolled up into 10s stats from 5s intervals through ops_buffer /** * The type name used within the Monitoring index to publish Kibana stats. * @type {string} @@ -30,6 +30,7 @@ export const KIBANA_STATS_TYPE = 'kibana_stats'; export const KIBANA_SETTINGS_TYPE = 'kibana_settings'; /** * The type name used within the Monitoring index to publish Kibana usage stats. + * NOTE: this string shows as-is in the stats API as a field name for the kibana usage stats * @type {string} */ export const KIBANA_USAGE_TYPE = 'kibana'; diff --git a/x-pack/plugins/monitoring/init.js b/x-pack/plugins/monitoring/init.js index f5d97ead958a7..84c23c3774b6f 100644 --- a/x-pack/plugins/monitoring/init.js +++ b/x-pack/plugins/monitoring/init.js @@ -26,14 +26,15 @@ import { * @param server {Object} HapiJS server instance */ export const init = (monitoringPlugin, server) => { + const kbnServer = monitoringPlugin.kbnServer; const config = server.config(); const { collectorSet } = server.usage; /* * Register collector objects for stats to show up in the APIs */ - collectorSet.register(getOpsStatsCollector(server)); + collectorSet.register(getOpsStatsCollector(server, kbnServer)); collectorSet.register(getKibanaUsageCollector(server)); - collectorSet.register(getSettingsCollector(server)); + collectorSet.register(getSettingsCollector(server, kbnServer)); /* * Instantiate and start the internal background task that calls collector @@ -53,7 +54,7 @@ export const init = (monitoringPlugin, server) => { } }); - const bulkUploader = initBulkUploader(monitoringPlugin.kbnServer, server); + const bulkUploader = initBulkUploader(kbnServer, server); const kibanaCollectionEnabled = config.get('xpack.monitoring.kibana.collection.enabled'); const { info: xpackMainInfo } = xpackMainPlugin; diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js index 17ebb46115605..0b69370b8e4c1 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js @@ -52,8 +52,7 @@ describe('BulkUploader', () => { ]); const uploader = new BulkUploader(server, { - interval: FETCH_INTERVAL, - combineTypes: noop, + interval: FETCH_INTERVAL }); uploader.start(collectors); @@ -82,16 +81,11 @@ describe('BulkUploader', () => { }); it('should run the bulk upload handler', done => { - const combineTypes = sinon.spy(data => { - return [data[0][0], { ...data[0][1], combined: true }]; - }); - const collectors = new MockCollectorSet(server, [ { fetch: () => ({ type: 'type_collector_test', result: { testData: 12345 } }) } ]); const uploader = new BulkUploader(server, { - interval: FETCH_INTERVAL, - combineTypes, + interval: FETCH_INTERVAL }); uploader.start(collectors); @@ -111,13 +105,6 @@ describe('BulkUploader', () => { 'Uploading bulk stats payload to the local cluster', ]); - // un-flattened - const combineCalls = combineTypes.getCalls(); - expect(combineCalls.length).to.be.greaterThan(0); // should be 1-2 fetch and combine cycles - expect(combineCalls[0].args).to.eql([ - [[{ index: { _type: 'type_collector_test' } }, { testData: 12345 }]], - ]); - done(); }, CHECK_DELAY); }); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/__tests__/get_collector_types_combiner.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js similarity index 70% rename from x-pack/plugins/monitoring/server/kibana_monitoring/lib/__tests__/get_collector_types_combiner.js rename to x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js index 4bf5c2a37c9a6..9b71d9417d521 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/__tests__/get_collector_types_combiner.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js @@ -4,13 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getCollectorTypesCombiner } from '../get_collector_types_combiner'; -import expect from 'expect.js'; +import { KIBANA_STATS_TYPE_MONITORING, KIBANA_USAGE_TYPE, KIBANA_SETTINGS_TYPE } from '../../common/constants'; +import { KIBANA_REPORTING_TYPE } from '../../../reporting/common/constants'; +import { BulkUploader } from './bulk_uploader'; const getInitial = () => { return [ [ - { 'index': { '_type': 'kibana_stats' } }, + { 'index': { '_type': KIBANA_STATS_TYPE_MONITORING } }, { 'host': 'tsullivan.local', 'concurrent_connections': 0, @@ -37,7 +38,7 @@ const getInitial = () => { } ], [ - { 'index': { '_type': 'kibana' } }, + { 'index': { '_type': KIBANA_USAGE_TYPE } }, { 'dashboard': { 'total': 0 }, 'visualization': { 'total': 0 }, @@ -47,7 +48,7 @@ const getInitial = () => { } ], [ - { 'index': { '_type': 'reporting_stats' } }, + { 'index': { '_type': KIBANA_REPORTING_TYPE } }, { 'available': true, 'enabled': false, @@ -63,17 +64,19 @@ const getInitial = () => { } ], [ - { 'index': { '_type': 'kibana_settings' } }, + { 'index': { '_type': KIBANA_SETTINGS_TYPE } }, { 'xpack': { 'defaultAdminEmail': 'tim@elastic.co' } } ] ]; }; +// TODO use jest snapshotting const getResult = () => { return [ [ { 'index': { '_type': 'kibana_stats' } }, { + 'host': 'tsullivan.local', 'concurrent_connections': 0, 'os': { 'load': { '1m': 2.28857421875, '5m': 2.45068359375, '15m': 2.29248046875 }, @@ -95,16 +98,6 @@ const getResult = () => { }, 'response_times': { 'average': 47, 'max': 47 }, 'timestamp': '2017-07-26T00:14:20.771Z', - 'kibana': { - 'uuid': '5b2de169-2785-441b-ae8c-186a1936b17d', - 'name': 'tsullivan.local', - 'index': '.kibana', - 'host': 'tsullivan.local', - 'transport_address': 'tsullivan.local:5601', - 'version': '6.0.0-beta1', - 'snapshot': false, - 'status': 'green' - }, 'usage': { 'dashboard': { 'total': 0 }, 'visualization': { 'total': 0 }, @@ -133,42 +126,18 @@ const getResult = () => { { 'index': { '_type': 'kibana_settings' } }, { 'xpack': { 'defaultAdminEmail': 'tim@elastic.co' }, - 'kibana': { - 'uuid': '5b2de169-2785-441b-ae8c-186a1936b17d', - 'name': 'tsullivan.local', - 'index': '.kibana', - 'host': 'tsullivan.local', - 'transport_address': 'tsullivan.local:5601', - 'version': '6.0.0-beta1', - 'snapshot': false, - 'status': 'green' - } } ] ]; }; -const kbnServerMock = {}; -const configMock = {}; -const sourceKibanaMock = () => ({ - uuid: '5b2de169-2785-441b-ae8c-186a1936b17d', - name: 'tsullivan.local', - index: '.kibana', - host: 'tsullivan.local', - transport_address: 'tsullivan.local:5601', - version: '6.0.0-beta1', - snapshot: false, - status: 'green' -}); - describe('Collector Types Combiner', () => { describe('with all the data types present', () => { it('provides settings, and combined stats/usage data', () => { // default gives all the data types const initial = getInitial(); - const combiner = getCollectorTypesCombiner(kbnServerMock, configMock, sourceKibanaMock); - const result = combiner(initial); - expect(result).to.eql(getResult()); + const result = BulkUploader.combineStatsLegacy(initial); + expect(result).toEqual(getResult()); }); }); describe('with settings data missing', () => { @@ -176,11 +145,10 @@ describe('Collector Types Combiner', () => { // default gives all the data types const initial = getInitial(); const trimmedInitial = [ initial[0], initial[1], initial[2] ]; // just stats, usage and reporting, no settings - const combiner = getCollectorTypesCombiner(kbnServerMock, configMock, sourceKibanaMock); - const result = combiner(trimmedInitial); + const result = BulkUploader.combineStatsLegacy(trimmedInitial); const expectedResult = getResult(); const trimmedExpectedResult = [ expectedResult[0] ]; // single combined item - expect(result).to.eql(trimmedExpectedResult); + expect(result).toEqual(trimmedExpectedResult); }); }); describe('with usage data missing', () => { @@ -188,12 +156,11 @@ describe('Collector Types Combiner', () => { // default gives all the data types const initial = getInitial(); const trimmedInitial = [ initial[0], initial[3] ]; // just stats and settings, no usage or reporting - const combiner = getCollectorTypesCombiner(kbnServerMock, configMock, sourceKibanaMock); - const result = combiner(trimmedInitial); + const result = BulkUploader.combineStatsLegacy(trimmedInitial); const expectedResult = getResult(); delete expectedResult[0][1].usage; // usage stats should not be present in the result const trimmedExpectedResult = [ expectedResult[0], expectedResult[1] ]; - expect(result).to.eql(trimmedExpectedResult); + expect(result).toEqual(trimmedExpectedResult); }); }); describe('with stats data missing', () => { @@ -201,11 +168,22 @@ describe('Collector Types Combiner', () => { // default gives all the data types const initial = getInitial(); const trimmedInitial = [ initial[3] ]; // just settings - const combiner = getCollectorTypesCombiner(kbnServerMock, configMock, sourceKibanaMock); - const result = combiner(trimmedInitial); + const result = BulkUploader.combineStatsLegacy(trimmedInitial); const expectedResult = getResult(); const trimmedExpectedResult = [ expectedResult[1] ]; // just settings - expect(result).to.eql(trimmedExpectedResult); + expect(result).toEqual(trimmedExpectedResult); }); }); + + it('throws an error if duplicate types are registered', () => { + const combineWithDuplicate = () => { + const initial = getInitial(); + const withDuplicate = [ initial[0] ].concat(initial); + return BulkUploader.combineStatsLegacy(withDuplicate); + }; + expect(combineWithDuplicate).toThrow( + 'Duplicate collector type identifiers found in payload! ' + + 'kibana_stats_monitoring,kibana_stats_monitoring,kibana,reporting,kibana_settings' + ); + }); }); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index de8fa0969c719..c95baa056c8d6 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -4,9 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import { isEmpty, flatten } from 'lodash'; +import { get, set, isEmpty, flatten, uniq } from 'lodash'; import { callClusterFactory } from '../../../xpack_main'; -import { LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG } from '../../common/constants'; +import { + LOGGING_TAG, + KIBANA_MONITORING_LOGGING_TAG, + KIBANA_STATS_TYPE_MONITORING, + KIBANA_SETTINGS_TYPE, + KIBANA_USAGE_TYPE, +} from '../../common/constants'; +import { KIBANA_REPORTING_TYPE } from '../../../reporting/common/constants'; import { sendBulkPayload, monitoringBulk, @@ -31,17 +38,13 @@ const LOGGING_TAGS = [LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG]; * @param {Object} xpackInfo server.plugins.xpack_main.info object */ export class BulkUploader { - constructor(server, { interval, combineTypes }) { + constructor(server, { interval }) { if (typeof interval !== 'number') { throw new Error('interval number of milliseconds is required'); } - if (typeof combineTypes !== 'function') { - throw new Error('combineTypes function is required'); - } this._timer = null; this._interval = interval; - this._combineTypes = combineTypes; this._log = { debug: message => server.log(['debug', ...LOGGING_TAGS], message), info: message => server.log(['info', ...LOGGING_TAGS], message), @@ -95,15 +98,12 @@ export class BulkUploader { */ async _fetchAndUpload(collectorSet) { const data = await collectorSet.bulkFetch(this._callClusterWithInternalUser); - const payload = data - .filter(d => Boolean(d) && !isEmpty(d.result)) - .map(({ result, type }) => [{ index: { _type: type } }, result]); + const payload = BulkUploader.toBulkUploadFormat(data); - if (payload.length > 0) { + if (payload) { try { - const combinedData = this._combineTypes(payload); // use the collector types combiner this._log.debug(`Uploading bulk stats payload to the local cluster`); - this._onPayload(flatten(combinedData)); + this._onPayload(payload); } catch (err) { this._log.warn(err.stack); this._log.warn(`Unable to bulk upload the stats payload to the local cluster`); @@ -116,4 +116,67 @@ export class BulkUploader { _onPayload(payload) { return sendBulkPayload(this._client, this._interval, payload); } + + /* + * Bulk stats are transformed into a bulk upload format + * Non-legacy transformation is done in CollectorSet.toApiStats + */ + static toBulkUploadFormat(uploadData) { + const payload = uploadData + .filter(d => Boolean(d) && !isEmpty(d.result)) + .map(({ result, type }) => [{ index: { _type: type } }, result]); + if (payload.length > 0) { + const combinedData = BulkUploader.combineStatsLegacy(payload); // arrange the usage data into the stats + return flatten(combinedData); + } + } + + static checkPayloadTypesUnique(payload) { + const ids = payload.map(item => item[0].index._type); + const uniques = uniq(ids); + if (ids.length !== uniques.length) { + throw new Error('Duplicate collector type identifiers found in payload! ' + ids.join(',')); + } + } + + static combineStatsLegacy(payload) { + BulkUploader.checkPayloadTypesUnique(payload); + + // default the item to [] to allow destructuring + const findItem = type => payload.find(item => get(item, '[0].index._type') === type) || []; + + // kibana usage and stats + let statsResult; + const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE_MONITORING); + const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE); + + if (statsHeader && statsPayload) { + statsHeader.index._type = 'kibana_stats'; // HACK to convert kibana_stats_monitoring to just kibana_stats for bwc + const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE); + const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null; + const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null; + statsResult = [ statsHeader, statsPayload ]; + if (kibanaUsage) { + set(statsResult, '[1].usage', kibanaUsage); + } + if (reportingUsage) { + set(statsResult, '[1].usage.xpack.reporting', reportingUsage); + } + } + + // kibana settings + let settingsResult; + const [ settingsHeader, settingsPayload ] = findItem(KIBANA_SETTINGS_TYPE); + if (settingsHeader && settingsPayload) { + settingsResult = [ settingsHeader, settingsPayload ]; + } + + // return new payload with the combined data + // adds usage data to stats data + // strips usage out as a top-level type + const result = [ statsResult, settingsResult ]; + + // remove result items that are undefined + return result.filter(Boolean); + } } diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js index cd6af97a029b4..1c79de29dea40 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js @@ -4,13 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { KIBANA_STATS_TYPE } from '../../../common/constants'; +import { KIBANA_STATS_TYPE_MONITORING } from '../../../common/constants'; import { opsBuffer } from './ops_buffer'; +import { getKibanaInfoForStats } from '../lib'; /* * Initialize a collector for Kibana Ops Stats */ -export function getOpsStatsCollector(server) { +export function getOpsStatsCollector(server, kbnServer) { let monitor; const buffer = opsBuffer(server); const onOps = event => buffer.push(event); @@ -44,8 +45,13 @@ export function getOpsStatsCollector(server) { const { collectorSet } = server.usage; return collectorSet.makeStatsCollector({ - type: KIBANA_STATS_TYPE, + type: KIBANA_STATS_TYPE_MONITORING, init: start, - fetch: buffer.flush + fetch: () => { + return { + kibana: getKibanaInfoForStats(server, kbnServer), + ...buffer.flush() + }; + } }); } diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js index 1a029169b365e..1382abb4ea293 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js @@ -7,6 +7,7 @@ import { get } from 'lodash'; import { XPACK_DEFAULT_ADMIN_EMAIL_UI_SETTING } from '../../../../../server/lib/constants'; import { KIBANA_SETTINGS_TYPE } from '../../../common/constants'; +import { getKibanaInfoForStats } from '../lib'; /* * Check if Cluster Alert email notifications is enabled in config @@ -53,7 +54,7 @@ export async function checkForEmailValue( } } -export function getSettingsCollector(server) { +export function getSettingsCollector(server, kbnServer) { const config = server.config(); const { collectorSet } = server.usage; @@ -78,7 +79,10 @@ export function getSettingsCollector(server) { // remember the current email so that we can mark it as successful if the bulk does not error out shouldUseNull = !!defaultAdminEmail; - return kibanaSettingsData; + return { + kibana: getKibanaInfoForStats(server, kbnServer), + ...kibanaSettingsData + }; } }); } diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/__tests__/event_roller.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/__tests__/event_roller.js index 55adc025ef8bd..bb4af020f4821 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/__tests__/event_roller.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/__tests__/event_roller.js @@ -17,10 +17,6 @@ const events = [ } }, responseTimes: { '5601': { avg: 5.213592233009709, max: 36 } }, - sockets: { - http: { total: 1, '169.254.169.254:80:': 1 }, - https: { total: 0 } - }, osload: [1.90380859375, 1.84033203125, 1.82666015625], osmem: { total: 17179869184, free: 613638144 }, osup: 4615, diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js index 524b3fd434c05..bb7683899200c 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js @@ -14,8 +14,6 @@ import { CloudDetector } from '../../../cloud'; * @return {Object} the revealed `push` and `flush` modules */ export function opsBuffer(server) { - let host = null; - // determine the cloud service in the background const cloudDetector = new CloudDetector(); cloudDetector.detectCloudService(); @@ -24,14 +22,12 @@ export function opsBuffer(server) { return { push(event) { - host = event.host; eventRoller.addEvent(event); server.log(['debug', LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG], 'Received Kibana Ops event data'); }, flush() { return { - host, cloud: cloudDetector.getCloudDetails(), ...eventRoller.flush() }; diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js index 5437babf8711c..6dcb0ae1fc076 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js @@ -5,7 +5,6 @@ */ import { BulkUploader } from './bulk_uploader'; -import { getCollectorTypesCombiner } from './lib'; /** * Initialize different types of Kibana Monitoring @@ -16,13 +15,11 @@ import { getCollectorTypesCombiner } from './lib'; * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core * @param {Object} server HapiJS server instance */ -export function initBulkUploader(kbnServer, server) { +export function initBulkUploader(_kbnServer, server) { const config = server.config(); const interval = config.get('xpack.monitoring.kibana.collection.interval'); return new BulkUploader(server, { - interval, - combineTypes: getCollectorTypesCombiner(kbnServer, config) + interval }); } - diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js deleted file mode 100644 index f50b080a9afc9..0000000000000 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { get, set, omit } from 'lodash'; -import { - KIBANA_STATS_TYPE, - KIBANA_SETTINGS_TYPE, - KIBANA_USAGE_TYPE, -} from '../../../common/constants'; -import { KIBANA_REPORTING_TYPE } from '../../../../reporting/common/constants'; -import { sourceKibana } from './source_kibana'; - -/* - * Combine stats collected from different sources into a single bulk payload. - * - * The ES Bulk Data Format is an array with 2 objects: - * - The first object is the header, it has a field for the action (index), and - * metadata of the document (_index, _type, _id). - * - The second object is the actual document to index. - * - * NOTE: https://github.com/elastic/kibana/issues/12504 asks that plugins have - * a way to register their own stats. It's not hard to move the stats collector - * methods under the ownership of the plugins that want it, but this module's - * behavior doesn't fit well with plugins registering their own stats. See the - * abstraction leak comments in the code. - * - * This module should go away when stats are collected by a Kibana metricbeat moduleset. - * - Individual plugin operational stats can be added to the `/stats?extended` API response. - * - Individual plugin usage stats can go into a new API similar to the `_xpack/usage` API in ES. - * - Each plugin will have its own top-level property in the responses for these APIs. - */ -export function getCollectorTypesCombiner(kbnServer, config, _sourceKibana = sourceKibana) { - return payload => { - // default the item to [] to allow destructuring - const findItem = type => payload.find(item => get(item, '[0].index._type') === type) || []; - - // kibana usage and stats - let statsResult; - const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE); - const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE); - - // sourceKibana uses "host" from the kibana stats payload - const host = get(statsPayload, 'host'); - const kibana = _sourceKibana(kbnServer, config, host); - - if (statsHeader && statsPayload) { - const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE); - const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null; - const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null; // this is an abstraction leak - statsResult = [ - statsHeader, - { - ...omit(statsPayload, 'host'), // remove the temp host field - kibana, - } - ]; - if (kibanaUsage) { - set(statsResult, '[1].usage', kibanaUsage); - } - if (reportingUsage) { - set(statsResult, '[1].usage.xpack.reporting', reportingUsage); // this is an abstraction leak - } - } - - // kibana settings - let settingsResult; - const [ settingsHeader, settingsPayload ] = findItem(KIBANA_SETTINGS_TYPE); - if (settingsHeader && settingsPayload) { - settingsResult = [ - settingsHeader, - { - ...settingsPayload, - kibana - } - ]; - } - - // return new payload with the combined data - // adds usage data to stats data - // strips usage out as a top-level type - const result = [ statsResult, settingsResult ]; - - // remove result items that are undefined - return result.filter(Boolean); - }; -} diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_kibana_info_for_stats.js similarity index 88% rename from x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js rename to x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_kibana_info_for_stats.js index 23f7c848d4bbd..7c9cb7dc530c9 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_kibana_info_for_stats.js @@ -12,21 +12,20 @@ const snapshotRegex = /-snapshot/i; * This provides a common structure to apply to all Kibana monitoring documents so that they can be commonly * searched, field-collapsed, and aggregated against. * - * 'sourceKibana' is akin to the `source_node` details in Elasticsearch nodes. - * * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core * @param {Object} config Server config * @param {String} host Kibana host * @return {Object} The object containing a "kibana" field and source instance details. */ -export function sourceKibana(kbnServer, config, host) { +export function getKibanaInfoForStats(server, kbnServer) { + const config = server.config(); const status = kbnServer.status.toJSON(); return { uuid: config.get('server.uuid'), name: config.get('server.name'), index: config.get('kibana.index'), - host, + host: config.get('server.host'), transport_address: `${config.get('server.host')}:${config.get('server.port')}`, version: kbnServer.version.replace(snapshotRegex, ''), snapshot: snapshotRegex.test(kbnServer.version), diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js index 2b81cd36fe302..56a2f48de88db 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js @@ -4,6 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -export { getCollectorTypesCombiner } from './get_collector_types_combiner'; export { sendBulkPayload } from './send_bulk_payload'; export { monitoringBulk } from './monitoring_bulk'; +export { getKibanaInfoForStats } from './get_kibana_info_for_stats'; diff --git a/x-pack/plugins/reporting/common/constants.js b/x-pack/plugins/reporting/common/constants.js index 6376c44042e25..09321bd8d3c3e 100644 --- a/x-pack/plugins/reporting/common/constants.js +++ b/x-pack/plugins/reporting/common/constants.js @@ -18,4 +18,4 @@ export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo'; * The type name used within the Monitoring index to publish reporting stats. * @type {string} */ -export const KIBANA_REPORTING_TYPE = 'reporting_stats'; +export const KIBANA_REPORTING_TYPE = 'reporting'; diff --git a/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js b/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js index 5f1f9e1b2b540..ea01d119efe7f 100644 --- a/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js +++ b/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js @@ -15,9 +15,11 @@ const getClusterUuid = async callCluster => { * @return {Object} data from usage stats collectors registered with Monitoring CollectorSet * @throws {Error} if the Monitoring CollectorSet is not ready */ -const getUsage = (callCluster, server) => { +const getUsage = async (callCluster, server) => { const { collectorSet } = server.usage; - return collectorSet.bulkFetchUsage(callCluster); + const usage = await collectorSet.bulkFetchUsage(callCluster); + const usageObject = collectorSet.toObject(usage); + return collectorSet.toApiFieldNames(usageObject); }; export function xpackUsageRoute(server) { diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js index 4068aa307b08a..7f105650141d9 100644 --- a/x-pack/test/api_integration/apis/index.js +++ b/x-pack/test/api_integration/apis/index.js @@ -10,5 +10,6 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./monitoring')); loadTestFile(require.resolve('./xpack_main')); loadTestFile(require.resolve('./logstash')); + loadTestFile(require.resolve('./kibana')); }); } diff --git a/x-pack/test/api_integration/apis/kibana/index.js b/x-pack/test/api_integration/apis/kibana/index.js new file mode 100644 index 0000000000000..d84ac9c75154a --- /dev/null +++ b/x-pack/test/api_integration/apis/kibana/index.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export default function ({ loadTestFile }) { + describe('kibana', () => { + loadTestFile(require.resolve('./stats')); + }); +} diff --git a/x-pack/test/api_integration/apis/kibana/stats/index.js b/x-pack/test/api_integration/apis/kibana/stats/index.js new file mode 100644 index 0000000000000..265a64dad7d13 --- /dev/null +++ b/x-pack/test/api_integration/apis/kibana/stats/index.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export default function ({ loadTestFile }) { + describe('stats', () => { + loadTestFile(require.resolve('./stats')); + }); +} diff --git a/x-pack/test/api_integration/apis/kibana/stats/stats.js b/x-pack/test/api_integration/apis/kibana/stats/stats.js new file mode 100644 index 0000000000000..e4a7c96328252 --- /dev/null +++ b/x-pack/test/api_integration/apis/kibana/stats/stats.js @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from 'expect.js'; + +export default function ({ getService }) { + const supertestNoAuth = getService('supertestWithoutAuth'); + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + describe('/api/stats', () => { + describe('operational stats and usage stats', () => { + before('load clusters archive', () => { + return esArchiver.load('discover'); + }); + + after('unload clusters archive', () => { + return esArchiver.unload('discover'); + }); + + describe('no auth', () => { + it('should return 200 and stats for no extended', async () => { + const { body } = await supertestNoAuth + .get('/api/stats') + .expect(200); + expect(body.kibana.uuid).to.eql('5b2de169-2785-441b-ae8c-186a1936b17d'); + expect(body.process.uptime_ms).to.be.greaterThan(0); + expect(body.os.uptime_ms).to.be.greaterThan(0); + expect(body.usage).to.be(undefined); + }); + + it('should return 401 for extended', async () => { + await supertestNoAuth + .get('/api/stats?extended') + .expect(401); // unauthorized + }); + }); + + describe('with auth', () => { + it('should return 200 and stats for no extended', async () => { + const { body } = await supertest + .get('/api/stats') + .expect(200); + expect(body.kibana.uuid).to.eql('5b2de169-2785-441b-ae8c-186a1936b17d'); + expect(body.process.uptime_ms).to.be.greaterThan(0); + expect(body.os.uptime_ms).to.be.greaterThan(0); + }); + + it('should return 200 for extended', async () => { + const { body } = await supertest + .get('/api/stats?extended') + .expect(200); + expect(body.kibana.uuid).to.eql('5b2de169-2785-441b-ae8c-186a1936b17d'); + expect(body.process.uptime_ms).to.be.greaterThan(0); + expect(body.os.uptime_ms).to.be.greaterThan(0); + expect(body.usage.kibana.index).to.be('.kibana'); + expect(body.usage.kibana.dashboard.total).to.be(0); + }); + }); + }); + }); +} diff --git a/x-pack/test/reporting/api/usage.js b/x-pack/test/reporting/api/usage.js index 018f40eef5118..8bfed1b2d5594 100644 --- a/x-pack/test/reporting/api/usage.js +++ b/x-pack/test/reporting/api/usage.js @@ -124,7 +124,11 @@ export default function ({ getService }) { }); }); - describe('deprecated API', () => { + /* Have to skip this test because the usage stats returned by the legacy + * endpoint aren't snake_cased in the legacy usage api. This will be + * completely removed in the next PR, when the legacy endpoint is removed + */ + describe.skip('deprecated API', () => { it('shows correct stats', async () => { const usage = await usageAPI.getUsageStatsFromDeprecatedPre64Endpoint(); diff --git a/x-pack/test/reporting/services/reporting_api.js b/x-pack/test/reporting/services/reporting_api.js index 361ed94c5066a..0c66a77a65fe2 100644 --- a/x-pack/test/reporting/services/reporting_api.js +++ b/x-pack/test/reporting/services/reporting_api.js @@ -98,8 +98,8 @@ export function ReportingAPIProvider({ getService }) { }, expectRecentPdfAppStats(stats, app, count) { - expect(stats.reporting.lastDay.printable_pdf.app[app]).to.be(count); - expect(stats.reporting.last7Days.printable_pdf.app[app]).to.be(count); + expect(stats.reporting.last_day.printable_pdf.app[app]).to.be(count); + expect(stats.reporting.last_7_days.printable_pdf.app[app]).to.be(count); }, expectAllTimePdfAppStats(stats, app, count) { @@ -107,8 +107,8 @@ export function ReportingAPIProvider({ getService }) { }, expectRecentPdfLayoutStats(stats, layout, count) { - expect(stats.reporting.lastDay.printable_pdf.layout[layout]).to.be(count); - expect(stats.reporting.last7Days.printable_pdf.layout[layout]).to.be(count); + expect(stats.reporting.last_day.printable_pdf.layout[layout]).to.be(count); + expect(stats.reporting.last_7_days.printable_pdf.layout[layout]).to.be(count); }, expectAllTimePdfLayoutStats(stats, layout, count) { @@ -116,8 +116,8 @@ export function ReportingAPIProvider({ getService }) { }, expectRecentJobTypeTotalStats(stats, jobType, count) { - expect(stats.reporting.lastDay[jobType].total).to.be(count); - expect(stats.reporting.last7Days[jobType].total).to.be(count); + expect(stats.reporting.last_day[jobType].total).to.be(count); + expect(stats.reporting.last_7_days[jobType].total).to.be(count); }, expectAllTimeJobTypeTotalStats(stats, jobType, count) {