Stats API: implement the "kibana status" spec from the Monitoring dat…

…a model for stats (elastic#20577) (elastic#20956)

* [Stats API] Set API field names per spec

* fix jest tests

* fix api integration test

* trash the original metrics collector

- constantly accumulating stats over time does not align with the existing behavior, which is to reset the stats to 0 whenever they are pulled

* move some logic out of the collector types combiner into inline

- change the signature of sourceKibana

* Make a new stats collector for the API

- to not clear the data when pulling via the api
- fetching is a read-only thing

* isolate data transforms for api data and upload data

* no static methods

* remove external in bytes

* remove the _stats prefix for kibana and reporting

* update jest test snapshot

* fix collector_types_combiner test

* fix usage api

* add test suite todo comment

* reduce some loc change

* roll back mysterious change

* reduce some more loc change

* comment correction

* reduce more loc change

* whitespace

* comment question

* fix cluster_uuid

* fix stats integration test

* fix bulk uploader test, combineTypes is no longer external

* very important comments about the current nature of stats represented and long-term goals

* add stats api tests with/without authentication

* fix more fields to match data model

* fix more tests

* fix jest test

* remove TODO

* remove sockets

* use snake_case for api field names

* restore accidental removal + copy/paste error

* sourceKibana -> getKibanaInfoForStats

* skip usage test on legacy endpoint

* fix api tests

* more comment

* stop putting a field in that used to be omitted

* fix the internal type to ID the usage data for bulk uploader

* correct the kibana usage type value, which is shown as-is in the API

* more fixes for the constants identifying collector types + test against duplicates

* add a comment on a hack, and a whitespace fix

src/server/kbn_server.js

@@ -26,8 +26,8 @@ import configSetupMixin from './config/setup';
 import httpMixin from './http';
 import { loggingMixin } from './logging';
 import warningsMixin from './warnings';
-import { statusMixin } from './status';
 import { usageMixin } from './usage';
+import { statusMixin } from './status';
 import pidMixin from './pid';
 import { configDeprecationWarningsMixin } from './config/deprecation_warnings';
 import configCompleteMixin from './config/complete';
@@ -68,8 +68,8 @@ export default class KbnServer {
       loggingMixin,
       configDeprecationWarningsMixin,
       warningsMixin,
-      statusMixin,
       usageMixin,
+      statusMixin,
 
       // writes pid file
       pidMixin,

src/server/status/collectors/get_ops_stats_collector.js

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import { KIBANA_STATS_TYPE } from '../constants';
+import { getKibanaInfoForStats } from '../lib';
+
+/*
+ * Initialize a collector for Kibana Ops Stats
+ *
+ * NOTE this collector's fetch method returns the latest stats from the
+ * Hapi/Good/Even-Better ops event listener. Therefore, the stats reset
+ * every 5 seconds (the default value of the ops.interval configuration
+ * setting). That makes it geared for providing the latest "real-time"
+ * stats. In the long-term, fetch should return stats that constantly
+ * accumulate over the server's uptime for better machine readability.
+ * Since the data is captured, timestamped and stored, the historical
+ * data can provide "real-time" stats by calculating a derivative of
+ * the metrics.
+ * See PR comment in https://github.com/elastic/kibana/pull/20577/files#r202416647
+ */
+export function getOpsStatsCollector(server, kbnServer) {
+  const { collectorSet } = server.usage;
+  return collectorSet.makeStatsCollector({
+    type: KIBANA_STATS_TYPE,
+    fetch: () => {
+      return {
+        kibana: getKibanaInfoForStats(server, kbnServer),
+        ...kbnServer.metrics // latest metrics captured from the ops event listener in src/server/status/index
+      };
+    }
+  });
+}

src/server/status/metrics_collector/index.js → src/server/status/collectors/index.js

@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export { MetricsCollector } from './metrics_collector';
+export { getOpsStatsCollector } from './get_ops_stats_collector';

src/server/status/constants.js

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const KIBANA_STATS_TYPE = 'kibana_stats'; // kibana stats per 5s intervals

src/server/status/index.js

@@ -18,30 +18,29 @@
  */
 
 import ServerStatus from './server_status';
-import { MetricsCollector } from './metrics_collector';
-import { Metrics } from './metrics_collector/metrics';
+import { Metrics } from './lib/metrics';
 import { registerStatusPage, registerStatusApi, registerStatsApi } from './routes';
+import { getOpsStatsCollector } from './collectors';
 
 export function statusMixin(kbnServer, server, config) {
-  const collector = new MetricsCollector(server, config);
   kbnServer.status = new ServerStatus(kbnServer.server);
 
+  const statsCollector = getOpsStatsCollector(server, kbnServer);
+  const { collectorSet } = server.usage;
+  collectorSet.register(statsCollector);
+
   const { ['even-better']: evenBetter } = server.plugins;
 
   if (evenBetter) {
     const metrics = new Metrics(config, server);
 
     evenBetter.monitor.on('ops', event => {
-      // for status API (to deprecate in next major)
-      metrics.capture(event).then(data => { kbnServer.metrics = data; });
-
-      // for metrics API (replacement API)
-      collector.collect(event); // collect() is async, but here we aren't depending on the return value
+      metrics.capture(event).then(data => { kbnServer.metrics = data; }); // captures (performs transforms on) the latest event data and stashes the metrics for status/stats API payload
     });
   }
 
   // init routes
   registerStatusPage(kbnServer, server, config);
   registerStatusApi(kbnServer, server, config);
-  registerStatsApi(kbnServer, server, config, collector);
+  registerStatsApi(kbnServer, server, config);
 }

src/server/status/lib/get_kibana_info_for_stats.js

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { get } from 'lodash';
+
+const snapshotRegex = /-snapshot/i;
+
+/**
+ * This provides a meta data attribute along with Kibana stats.
+ *
+ * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core
+ * @param {Object} config Server config
+ * @param {String} host Kibana host
+ * @return {Object} The object containing a "kibana" field and source instance details.
+ */
+export function getKibanaInfoForStats(server, kbnServer) {
+  const config = server.config();
+  const status = kbnServer.status.toJSON();
+
+  return {
+    uuid: config.get('server.uuid'),
+    name: config.get('server.name'),
+    index: config.get('kibana.index'),
+    host: config.get('server.host'),
+    transport_address: `${config.get('server.host')}:${config.get('server.port')}`,
+    version: kbnServer.version.replace(snapshotRegex, ''),
+    snapshot: snapshotRegex.test(kbnServer.version),
+    status: get(status, 'overall.state')
+  };
+}

src/server/status/lib/index.js

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { getKibanaInfoForStats } from './get_kibana_info_for_stats';

src/server/status/metrics_collector/metrics.js → src/server/status/lib/metrics.js

@@ -18,6 +18,7 @@
  */
 
 import os from 'os';
+import v8 from 'v8';
 import { get, isObject, merge } from 'lodash';
 import { keysToSnakeCaseShallow } from '../../../utils/case_conversion';
 import { getAllStats as cGroupStats } from './cgroup';
@@ -32,19 +33,17 @@ export class Metrics {
   static getStubMetrics() {
     return {
       process: {
-        mem: {}
+        memory: {
+          heap: {}
+        }
       },
       os: {
         cpu: {},
-        mem: {}
+        memory: {}
       },
       response_times: {},
       requests: {
         status_codes: {}
-      },
-      sockets: {
-        http: {},
-        https: {}
       }
     };
   }
@@ -56,53 +55,52 @@ export class Metrics {
 
     const metrics = {
       last_updated: timestamp,
-      collection_interval_in_millis: this.config.get('ops.interval'),
-      uptime_in_millis: event.process.uptime_ms, // TODO: deprecate this field, data should only have process.uptime_ms
+      collection_interval_in_millis: this.config.get('ops.interval')
     };
 
     return merge(metrics, event, cgroup);
   }
 
   captureEvent(hapiEvent) {
+    const heapStats = v8.getHeapStatistics();
     const port = this.config.get('server.port');
-
     const avgInMillis = get(hapiEvent, ['responseTimes', port, 'avg']); // sadly, it's possible for this to be NaN
     const maxInMillis = get(hapiEvent, ['responseTimes', port, 'max']);
 
     return {
       process: {
-        mem: {
-          // https://nodejs.org/docs/latest-v8.x/api/process.html#process_process_memoryusage
-          heap_max_in_bytes: get(hapiEvent, 'psmem.heapTotal'),
-          heap_used_in_bytes: get(hapiEvent, 'psmem.heapUsed'),
+        memory: {
+          heap: {
+            // https://nodejs.org/docs/latest-v8.x/api/process.html#process_process_memoryusage
+            total_in_bytes: get(hapiEvent, 'psmem.heapTotal'),
+            used_in_bytes: get(hapiEvent, 'psmem.heapUsed'),
+            size_limit: heapStats.heap_size_limit
+          },
           resident_set_size_in_bytes: get(hapiEvent, 'psmem.rss'),
-          external_in_bytes: get(hapiEvent, 'psmem.external')
         },
+        event_loop_delay: get(hapiEvent, 'psdelay'),
         pid: process.pid,
-        uptime_ms: process.uptime() * 1000
+        uptime_in_millis: process.uptime() * 1000
       },
       os: {
-        cpu: {
-          load_average: {
-            '1m': get(hapiEvent, 'osload.0'),
-            '5m': get(hapiEvent, 'osload.1'),
-            '15m': get(hapiEvent, 'osload.2')
-          }
+        load: {
+          '1m': get(hapiEvent, 'osload.0'),
+          '5m': get(hapiEvent, 'osload.1'),
+          '15m': get(hapiEvent, 'osload.2')
         },
-        mem: {
+        memory: {
+          total_in_bytes: os.totalmem(),
           free_in_bytes: os.freemem(),
-          total_in_bytes: os.totalmem()
-        }
+          used_in_bytes: get(hapiEvent, 'osmem.total') - get(hapiEvent, 'osmem.free')
+        },
+        uptime_in_millis: os.uptime() * 1000
       },
       response_times: {
-        // TODO: rename to use `_ms` suffix per beats naming conventions
         avg_in_millis: isNaN(avgInMillis) ? undefined : avgInMillis, // convert NaN to undefined
         max_in_millis: maxInMillis
       },
       requests: keysToSnakeCaseShallow(get(hapiEvent, ['requests', port])),
       concurrent_connections: get(hapiEvent, ['concurrents', port]),
-      sockets: get(hapiEvent, 'sockets'),
-      event_loop_delay: get(hapiEvent, 'psdelay')
     };
   }