From cd83e739ec5a960210f070aaec2feff797ca74ee Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 5 Jul 2018 13:36:36 -0700
Subject: [PATCH 01/42] [Stats API] Set API field names per spec

---
 src/server/kbn_server.js                      |  4 +-
 src/server/status/index.js                    |  7 +--
 src/server/status/lib/index.js                | 20 ++++++++
 src/server/status/lib/set_api_field_names.js  | 41 ++++++++++++++++
 .../status/metrics_collector/metrics.js       |  7 ++-
 .../metrics_collector/metrics_collector.js    | 31 +++++++-----
 .../status/routes/api/register_stats.js       | 47 +++++++++++++------
 .../status/routes/api/register_status.js      |  5 +-
 src/server/usage/classes/collector_set.js     | 16 ++++---
 .../server/kibana_monitoring/init.js          |  3 +-
 .../server/routes/api/v1/xpack_usage.js       |  5 +-
 11 files changed, 136 insertions(+), 50 deletions(-)
 create mode 100644 src/server/status/lib/index.js
 create mode 100644 src/server/status/lib/set_api_field_names.js

diff --git a/src/server/kbn_server.js b/src/server/kbn_server.js
index 7e9895d4a3afd..014cfbb845307 100644
--- a/src/server/kbn_server.js
+++ b/src/server/kbn_server.js
@@ -26,8 +26,8 @@ import configSetupMixin from './config/setup';
 import httpMixin from './http';
 import { loggingMixin } from './logging';
 import warningsMixin from './warnings';
-import { statusMixin } from './status';
 import { usageMixin } from './usage';
+import { statusMixin } from './status';
 import pidMixin from './pid';
 import { configDeprecationWarningsMixin } from './config/deprecation_warnings';
 import configCompleteMixin from './config/complete';
@@ -68,8 +68,8 @@ export default class KbnServer {
       loggingMixin,
       configDeprecationWarningsMixin,
       warningsMixin,
-      statusMixin,
       usageMixin,
+      statusMixin,
 
       // writes pid file
       pidMixin,
diff --git a/src/server/status/index.js b/src/server/status/index.js
index 4367353f70a6f..6dc45655fd3e5 100644
--- a/src/server/status/index.js
+++ b/src/server/status/index.js
@@ -32,11 +32,8 @@ export function statusMixin(kbnServer, server, config) {
     const metrics = new Metrics(config, server);
 
     evenBetter.monitor.on('ops', event => {
-      // for status API (to deprecate in next major)
-      metrics.capture(event).then(data => { kbnServer.metrics = data; });
-
-      // for metrics API (replacement API)
-      collector.collect(event); // collect() is async, but here we aren't depending on the return value
+      metrics.capture(event).then(data => { kbnServer.metrics = data; }); // for status API (to deprecate in next major)
+      collector.collect(event); // for metrics API (replacement API)
     });
   }
 
diff --git a/src/server/status/lib/index.js b/src/server/status/lib/index.js
new file mode 100644
index 0000000000000..abab4556677b5
--- /dev/null
+++ b/src/server/status/lib/index.js
@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { setApiFieldNames } from './set_api_field_names';
diff --git a/src/server/status/lib/set_api_field_names.js b/src/server/status/lib/set_api_field_names.js
new file mode 100644
index 0000000000000..590eec5c40a9a
--- /dev/null
+++ b/src/server/status/lib/set_api_field_names.js
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+function getValueOrRecurse(value) {
+  if (value == null || typeof value !== 'object') {
+    return value;
+  } else {
+    return setApiFieldNames(value); // recurse
+  }
+}
+
+export function setApiFieldNames(apiData) {
+  return Object.keys(apiData).reduce((accum, currName) => {
+    const value = apiData[currName];
+
+    let newName = currName;
+    newName = newName.replace('_in_bytes', '_bytes');
+    newName = newName.replace('_in_millis', '_ms');
+
+    return {
+      ...accum,
+      [newName]: getValueOrRecurse(value),
+    };
+  }, {});
+}
diff --git a/src/server/status/metrics_collector/metrics.js b/src/server/status/metrics_collector/metrics.js
index f6499d44b32ee..bc404ae628b5f 100644
--- a/src/server/status/metrics_collector/metrics.js
+++ b/src/server/status/metrics_collector/metrics.js
@@ -56,8 +56,7 @@ export class Metrics {
 
     const metrics = {
       last_updated: timestamp,
-      collection_interval_in_millis: this.config.get('ops.interval'),
-      uptime_in_millis: event.process.uptime_ms, // TODO: deprecate this field, data should only have process.uptime_ms
+      collection_interval_in_millis: this.config.get('ops.interval')
     };
 
     return merge(metrics, event, cgroup);
@@ -92,10 +91,10 @@ export class Metrics {
         mem: {
           free_in_bytes: os.freemem(),
           total_in_bytes: os.totalmem()
-        }
+        },
+        uptime_ms: os.uptime() * 1000
       },
       response_times: {
-        // TODO: rename to use `_ms` suffix per beats naming conventions
         avg_in_millis: isNaN(avgInMillis) ? undefined : avgInMillis, // convert NaN to undefined
         max_in_millis: maxInMillis
       },
diff --git a/src/server/status/metrics_collector/metrics_collector.js b/src/server/status/metrics_collector/metrics_collector.js
index 1d3ee4fb63b48..3c1b3be7de914 100644
--- a/src/server/status/metrics_collector/metrics_collector.js
+++ b/src/server/status/metrics_collector/metrics_collector.js
@@ -29,17 +29,18 @@ const matchSnapshot = /-SNAPSHOT$/;
 export class MetricsCollector {
   constructor(server, config) {
 
-    // NOTE we need access to config every time this is used because uuid is managed by the kibana core_plugin, which is initialized AFTER kbn_server
-    this._getBaseStats = () => ({
-      name: config.get('server.name'),
-      uuid: config.get('server.uuid'),
-      version: {
-        number: config.get('pkg.version').replace(matchSnapshot, ''),
-        build_hash: config.get('pkg.buildSha'),
-        build_number: config.get('pkg.buildNum'),
-        build_snapshot: matchSnapshot.test(config.get('pkg.version'))
-      }
-    });
+    // NOTE we need access to config every time this is used because uuid is
+    // managed by the kibana core_plugin, which is initialized AFTER kbn_server
+    this._getKibanaStats = () => {
+      return {
+        name: config.get('server.name'),
+        host: config.get('server.host'),
+        uuid: config.get('server.uuid'),
+        transport_address: `${config.get('server.host')}:${config.get('server.port')}`,
+        version: config.get('pkg.version'),
+        snapshot: matchSnapshot.test(config.get('pkg.version')),
+      };
+    };
 
     this._stats = Metrics.getStubMetrics();
     this._metrics = new Metrics(config, server); // TODO: deprecate status API that uses Metrics class, move it this module, fix the order of its constructor params
@@ -104,9 +105,13 @@ export class MetricsCollector {
     return stats;
   }
 
-  getStats() {
+  getStats(kbnServer) {
+    const status = kbnServer.status.toJSON();
     return {
-      ...this._getBaseStats(),
+      kibana: {
+        ...this._getKibanaStats(),
+        status: status.overall.state
+      },
       ...this._stats
     };
   }
diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js
index 296fb532573c9..0548b53d3e0ce 100644
--- a/src/server/status/routes/api/register_stats.js
+++ b/src/server/status/routes/api/register_stats.js
@@ -19,10 +19,35 @@
 
 import Joi from 'joi';
 import { wrapAuthConfig } from '../../wrap_auth_config';
+import { setApiFieldNames } from '../../lib';
+
+async function getExtended(req, server) {
+  const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('admin'); // admin cluster, get info on internal system
+  const callCluster = (...args) => callWithRequest(req, ...args);
+
+  let clusterUuid;
+  try {
+    const { cluster_uuid: uuid } = await callCluster('info', { filterPath: 'cluster_uuid', });
+    clusterUuid = uuid;
+  } catch (err) {
+    clusterUuid = undefined; // fallback from anonymous access or auth failure, redundant for explicitness
+  }
+
+  let usage;
+  try {
+    const { collectorSet } = server.usage;
+    const usageRaw = await collectorSet.bulkFetchUsage(callCluster);
+    usage = collectorSet.summarizeStats(usageRaw);
+  } catch (err) {
+    usage = undefined;
+  }
+
+  return { clusterUuid, usage };
+}
 
 /*
  * API for Kibana meta info and accumulated operations stats
- * Including ?extended in the query string fetches Elasticsearch cluster_uuid
+ * Including ?extended in the query string fetches Elasticsearch cluster_uuid and server.usage.collectorSet data
  * - Requests to set isExtended = true
  *      GET /api/stats?extended=true
  *      GET /api/stats?extended
@@ -48,22 +73,16 @@ export function registerStatsApi(kbnServer, server, config, collector) {
         const isExtended = extended !== undefined && extended !== 'false';
 
         let clusterUuid;
+        let usage;
         if (isExtended) {
-          try {
-            const { callWithRequest, } = server.plugins.elasticsearch.getCluster('data');
-            const { cluster_uuid: uuid } = await callWithRequest(req, 'info', { filterPath: 'cluster_uuid', });
-            clusterUuid = uuid;
-          } catch (err) {
-            clusterUuid = undefined; // fallback from anonymous access or auth failure, redundant for explicitness
-          }
+          ({ clusterUuid, usage } = await getExtended(req, server));
         }
 
-        const stats = {
-          cluster_uuid: clusterUuid, // serialization makes an undefined get stripped out, as undefined isn't a JSON type
-          status: kbnServer.status.toJSON(),
-          ...collector.getStats(),
-        };
-
+        const stats = setApiFieldNames({
+          ...collector.getStats(kbnServer),
+          cluster_uuid: clusterUuid,
+          usage,
+        });
         reply(stats);
       },
     })
diff --git a/src/server/status/routes/api/register_status.js b/src/server/status/routes/api/register_status.js
index 98c454de4f58d..64f23170a44fb 100644
--- a/src/server/status/routes/api/register_status.js
+++ b/src/server/status/routes/api/register_status.js
@@ -41,7 +41,10 @@ export function registerStatusApi(kbnServer, server, config) {
           build_snapshot: matchSnapshot.test(config.get('pkg.version'))
         },
         status: kbnServer.status.toJSON(),
-        metrics: kbnServer.metrics
+        metrics: {
+          ...kbnServer.metrics,
+          uptime_in_millis: process.uptime() * 1000
+        }
       };
 
       return reply(status);
diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js
index 4ce2a88776326..53ae4c77ea0d8 100644
--- a/src/server/usage/classes/collector_set.js
+++ b/src/server/usage/classes/collector_set.js
@@ -89,14 +89,16 @@ export class CollectorSet {
 
   async bulkFetchUsage(callCluster) {
     const usageCollectors = this._collectors.filter(c => c instanceof UsageCollector);
-    const bulk = await this.bulkFetch(callCluster, usageCollectors);
+    return this.bulkFetch(callCluster, usageCollectors);
 
-    // summarize each type of stat
-    return bulk.reduce((accumulatedStats, currentStat) => {
-      /* Suffix removal is a temporary hack: some types have `_stats` suffix
-       * because of how monitoring bulk upload needed to combine types. It can
-       * be removed when bulk upload goes away
-       */
+  }
+
+  /*
+   * Summarize the data returned by bulk fetching into a simpler format
+   */
+  summarizeStats(statsData) {
+    return statsData.reduce((accumulatedStats, currentStat) => {
+      // `_stats` Suffix removal
       const statType = currentStat.type.replace('_stats', '');
       return {
         ...accumulatedStats,
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js
index 5437babf8711c..3e136c5226cbc 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js
@@ -10,7 +10,7 @@ import { getCollectorTypesCombiner } from './lib';
 /**
  * Initialize different types of Kibana Monitoring
  * TODO: remove this in 7.0
- * - Ops Events - essentially Kibana's /api/status
+ * - Ops Events - essentially Kibana's /api/status (non-rolled up)
  * - Usage Stats - essentially Kibana's /api/stats
  * - Kibana Settings - select uiSettings
  * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core
@@ -25,4 +25,3 @@ export function initBulkUploader(kbnServer, server) {
     combineTypes: getCollectorTypesCombiner(kbnServer, config)
   });
 }
-
diff --git a/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js b/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js
index 5f1f9e1b2b540..a1e6b4fd2ab9a 100644
--- a/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js
+++ b/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js
@@ -15,9 +15,10 @@ const getClusterUuid = async callCluster => {
  * @return {Object} data from usage stats collectors registered with Monitoring CollectorSet
  * @throws {Error} if the Monitoring CollectorSet is not ready
  */
-const getUsage = (callCluster, server) => {
+const getUsage = async (callCluster, server) => {
   const { collectorSet } = server.usage;
-  return collectorSet.bulkFetchUsage(callCluster);
+  const usage = await collectorSet.bulkFetchUsage(callCluster);
+  return collectorSet.summarizeStats(usage);
 };
 
 export function xpackUsageRoute(server) {

From acba93c0d7bddaf3f45547165eab818ac783a8e1 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Wed, 11 Jul 2018 10:00:14 -0700
Subject: [PATCH 02/42] fix jest tests

---
 .../__snapshots__/metrics.test.js.snap        | 114 ++++++++++++++++++
 .../metrics_collector.test.js.snap            |  93 +++++++-------
 .../status/metrics_collector/metrics.test.js  |  72 ++---------
 .../metrics_collector.test.js                 |  21 +++-
 4 files changed, 187 insertions(+), 113 deletions(-)
 create mode 100644 src/server/status/metrics_collector/__snapshots__/metrics.test.js.snap

diff --git a/src/server/status/metrics_collector/__snapshots__/metrics.test.js.snap b/src/server/status/metrics_collector/__snapshots__/metrics.test.js.snap
new file mode 100644
index 0000000000000..72f148f69dbe6
--- /dev/null
+++ b/src/server/status/metrics_collector/__snapshots__/metrics.test.js.snap
@@ -0,0 +1,114 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Metrics capture merges all metrics 1`] = `
+Object {
+  "a": Array [
+    Object {
+      "b": 2,
+      "c": 3,
+    },
+    Object {
+      "d": 4,
+      "e": 5,
+    },
+  ],
+  "collection_interval_in_millis": 5000,
+  "last_updated": "2017-04-14T18:35:41.534Z",
+  "process": Object {
+    "uptime_ms": 1980,
+  },
+}
+`;
+
+exports[`Metrics captureEvent parses event with missing fields / NaN for responseTimes.avg 1`] = `
+Object {
+  "concurrent_connections": undefined,
+  "event_loop_delay": undefined,
+  "os": Object {
+    "cpu": Object {
+      "load_average": Object {
+        "15m": undefined,
+        "1m": undefined,
+        "5m": undefined,
+      },
+    },
+    "mem": Object {
+      "free_in_bytes": 12,
+      "total_in_bytes": 24,
+    },
+    "uptime_ms": 12000000,
+  },
+  "process": Object {
+    "mem": Object {
+      "external_in_bytes": undefined,
+      "heap_max_in_bytes": undefined,
+      "heap_used_in_bytes": undefined,
+      "resident_set_size_in_bytes": undefined,
+    },
+    "pid": 8675309,
+    "uptime_ms": 5000000,
+  },
+  "requests": Object {
+    "disconnects": 0,
+    "status_codes": Object {
+      "200": 22,
+    },
+    "total": 22,
+  },
+  "response_times": Object {
+    "avg_in_millis": undefined,
+    "max_in_millis": 4,
+  },
+  "sockets": undefined,
+}
+`;
+
+exports[`Metrics captureEvent parses the hapi event 1`] = `
+Object {
+  "concurrent_connections": 0,
+  "event_loop_delay": 1.6091690063476562,
+  "os": Object {
+    "cpu": Object {
+      "load_average": Object {
+        "15m": 1.89794921875,
+        "1m": 2.20751953125,
+        "5m": 2.02294921875,
+      },
+    },
+    "mem": Object {
+      "free_in_bytes": 12,
+      "total_in_bytes": 24,
+    },
+    "uptime_ms": 12000000,
+  },
+  "process": Object {
+    "mem": Object {
+      "external_in_bytes": 1779619,
+      "heap_max_in_bytes": 168194048,
+      "heap_used_in_bytes": 130553400,
+      "resident_set_size_in_bytes": 193716224,
+    },
+    "pid": 8675309,
+    "uptime_ms": 5000000,
+  },
+  "requests": Object {
+    "disconnects": 0,
+    "status_codes": Object {
+      "200": 22,
+    },
+    "total": 22,
+  },
+  "response_times": Object {
+    "avg_in_millis": 1.8636363636363635,
+    "max_in_millis": 4,
+  },
+  "sockets": Object {
+    "http": Object {
+      "total": 0,
+    },
+    "https": Object {
+      "total": 0,
+    },
+  },
+}
+`;
diff --git a/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap b/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap
index 06b9b275f7200..014876004c3a7 100644
--- a/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap
+++ b/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap
@@ -5,8 +5,16 @@ Object {
   "collection_interval_in_millis": "test-123",
   "concurrent_connections": 0,
   "event_loop_delay": 0.3764979839324951,
+  "kibana": Object {
+    "host": "test-123",
+    "name": "test-123",
+    "snapshot": false,
+    "status": "green",
+    "transport_address": "test-123:3000",
+    "uuid": "test-123",
+    "version": "test-123",
+  },
   "last_updated": "2018-04-19T21:50:54.366Z",
-  "name": "test-123",
   "os": Object {
     "cpu": Object {
       "load_average": Object {
@@ -19,6 +27,7 @@ Object {
       "free_in_bytes": 12,
       "total_in_bytes": 24,
     },
+    "uptime_ms": NaN,
   },
   "process": Object {
     "mem": Object {
@@ -49,14 +58,6 @@ Object {
       "total": 0,
     },
   },
-  "uptime_in_millis": 6666000,
-  "uuid": "test-123",
-  "version": Object {
-    "build_hash": "test-123",
-    "build_number": "test-123",
-    "build_snapshot": false,
-    "number": "test-123",
-  },
 }
 `;
 
@@ -65,8 +66,16 @@ Object {
   "collection_interval_in_millis": "test-123",
   "concurrent_connections": 0,
   "event_loop_delay": 0.7529959678649902,
+  "kibana": Object {
+    "host": "test-123",
+    "name": "test-123",
+    "snapshot": false,
+    "status": "green",
+    "transport_address": "test-123:3000",
+    "uuid": "test-123",
+    "version": "test-123",
+  },
   "last_updated": "2018-04-19T21:50:54.366Z",
-  "name": "test-123",
   "os": Object {
     "cpu": Object {
       "load_average": Object {
@@ -79,6 +88,7 @@ Object {
       "free_in_bytes": 12,
       "total_in_bytes": 24,
     },
+    "uptime_ms": NaN,
   },
   "process": Object {
     "mem": Object {
@@ -109,14 +119,6 @@ Object {
       "total": 0,
     },
   },
-  "uptime_in_millis": 6666000,
-  "uuid": "test-123",
-  "version": Object {
-    "build_hash": "test-123",
-    "build_number": "test-123",
-    "build_snapshot": false,
-    "number": "test-123",
-  },
 }
 `;
 
@@ -125,8 +127,16 @@ Object {
   "collection_interval_in_millis": "test-123",
   "concurrent_connections": 0,
   "event_loop_delay": 1.1294939517974854,
+  "kibana": Object {
+    "host": "test-123",
+    "name": "test-123",
+    "snapshot": false,
+    "status": "green",
+    "transport_address": "test-123:3000",
+    "uuid": "test-123",
+    "version": "test-123",
+  },
   "last_updated": "2018-04-19T21:50:54.366Z",
-  "name": "test-123",
   "os": Object {
     "cpu": Object {
       "load_average": Object {
@@ -139,6 +149,7 @@ Object {
       "free_in_bytes": 12,
       "total_in_bytes": 24,
     },
+    "uptime_ms": NaN,
   },
   "process": Object {
     "mem": Object {
@@ -169,14 +180,6 @@ Object {
       "total": 0,
     },
   },
-  "uptime_in_millis": 6666000,
-  "uuid": "test-123",
-  "version": Object {
-    "build_hash": "test-123",
-    "build_number": "test-123",
-    "build_snapshot": false,
-    "number": "test-123",
-  },
 }
 `;
 
@@ -185,8 +188,16 @@ Object {
   "collection_interval_in_millis": "test-123",
   "concurrent_connections": 0,
   "event_loop_delay": 0.33843398094177246,
+  "kibana": Object {
+    "host": "test-123",
+    "name": "test-123",
+    "snapshot": false,
+    "status": "green",
+    "transport_address": "test-123:3000",
+    "uuid": "test-123",
+    "version": "test-123",
+  },
   "last_updated": "2018-04-19T21:50:54.366Z",
-  "name": "test-123",
   "os": Object {
     "cpu": Object {
       "load_average": Object {
@@ -199,6 +210,7 @@ Object {
       "free_in_bytes": 12,
       "total_in_bytes": 24,
     },
+    "uptime_ms": NaN,
   },
   "process": Object {
     "mem": Object {
@@ -229,20 +241,20 @@ Object {
       "total": 0,
     },
   },
-  "uptime_in_millis": 6666000,
-  "uuid": "test-123",
-  "version": Object {
-    "build_hash": "test-123",
-    "build_number": "test-123",
-    "build_snapshot": false,
-    "number": "test-123",
-  },
 }
 `;
 
 exports[`Metrics Collector initialize should return stub metrics 1`] = `
 Object {
-  "name": "test-123",
+  "kibana": Object {
+    "host": "test-123",
+    "name": "test-123",
+    "snapshot": false,
+    "status": "green",
+    "transport_address": "test-123:3000",
+    "uuid": "test-123",
+    "version": "test-123",
+  },
   "os": Object {
     "cpu": Object {},
     "mem": Object {},
@@ -258,12 +270,5 @@ Object {
     "http": Object {},
     "https": Object {},
   },
-  "uuid": "test-123",
-  "version": Object {
-    "build_hash": "test-123",
-    "build_number": "test-123",
-    "build_snapshot": false,
-    "number": "test-123",
-  },
 }
 `;
diff --git a/src/server/status/metrics_collector/metrics.test.js b/src/server/status/metrics_collector/metrics.test.js
index 24459c28e981d..ed82a15db3a1b 100644
--- a/src/server/status/metrics_collector/metrics.test.js
+++ b/src/server/status/metrics_collector/metrics.test.js
@@ -23,10 +23,13 @@ jest.mock('fs', () => ({
 
 jest.mock('os', () => ({
   freemem: jest.fn(),
-  totalmem: jest.fn()
+  totalmem: jest.fn(),
+  uptime: jest.fn()
 }));
 
-jest.mock('process');
+jest.mock('process', () => ({
+  uptime: jest.fn()
+}));
 
 import fs from 'fs';
 import os from 'os';
@@ -69,17 +72,13 @@ describe('Metrics', function () {
       sinon.stub(Date.prototype, 'toISOString').returns('2017-04-14T18:35:41.534Z');
 
       const capturedMetrics = await metrics.capture();
-      expect(capturedMetrics).toEqual({
-        last_updated: '2017-04-14T18:35:41.534Z',
-        collection_interval_in_millis: 5000,
-        uptime_in_millis: 1980,
-        a: [ { b: 2, c: 3 }, { d: 4, e: 5 } ], process: { uptime_ms: 1980 }
-      });
+      expect(capturedMetrics).toMatchSnapshot();
     });
   });
 
   describe('captureEvent', () => {
     it('parses the hapi event', () => {
+      sinon.stub(os, 'uptime').returns(12000);
       sinon.stub(process, 'uptime').returns(5000);
 
       os.freemem.mockImplementation(() => 12);
@@ -106,52 +105,7 @@ describe('Metrics', function () {
         'host': 'blahblah.local'
       };
 
-      expect(metrics.captureEvent(hapiEvent)).toEqual({
-        'concurrent_connections': 0,
-        'event_loop_delay': 1.6091690063476562,
-        'os': {
-          'cpu': {
-            'load_average': {
-              '15m': 1.89794921875,
-              '1m': 2.20751953125,
-              '5m': 2.02294921875
-            }
-          },
-          'mem': {
-            'free_in_bytes': 12,
-            'total_in_bytes': 24,
-          },
-        },
-        'process': {
-          'mem': {
-            'external_in_bytes': 1779619,
-            'heap_max_in_bytes': 168194048,
-            'heap_used_in_bytes': 130553400,
-            'resident_set_size_in_bytes': 193716224,
-          },
-          'pid': 8675309,
-          'uptime_ms': 5000000
-        },
-        'requests': {
-          'disconnects': 0,
-          'status_codes': {
-            '200': 22
-          },
-          'total': 22
-        },
-        'response_times': {
-          'avg_in_millis': 1.8636363636363635,
-          'max_in_millis': 4
-        },
-        'sockets': {
-          'http': {
-            'total': 0
-          },
-          'https': {
-            'total': 0
-          }
-        }
-      });
+      expect(metrics.captureEvent(hapiEvent)).toMatchSnapshot();
     });
 
     it('parses event with missing fields / NaN for responseTimes.avg', () => {
@@ -163,15 +117,7 @@ describe('Metrics', function () {
         host: 'blahblah.local',
       };
 
-      expect(metrics.captureEvent(hapiEvent)).toEqual({
-        process: { mem: {}, pid: 8675309, uptime_ms: 5000000 },
-        os: {
-          cpu: { load_average: {} },
-          mem: { free_in_bytes: 12, total_in_bytes: 24 },
-        },
-        response_times: { max_in_millis: 4 },
-        requests: { total: 22, disconnects: 0, status_codes: { '200': 22 } },
-      });
+      expect(metrics.captureEvent(hapiEvent)).toMatchSnapshot();
     });
   });
 
diff --git a/src/server/status/metrics_collector/metrics_collector.test.js b/src/server/status/metrics_collector/metrics_collector.test.js
index e6107d6b3e3c0..fbecb65dee61c 100644
--- a/src/server/status/metrics_collector/metrics_collector.test.js
+++ b/src/server/status/metrics_collector/metrics_collector.test.js
@@ -19,7 +19,8 @@
 
 jest.mock('os', () => ({
   freemem: jest.fn(),
-  totalmem: jest.fn()
+  totalmem: jest.fn(),
+  uptime: jest.fn()
 }));
 
 const mockProcessUptime = jest.fn().mockImplementation(() => 6666);
@@ -38,11 +39,19 @@ const mockConfig = {
 mockConfig.get.returns('test-123');
 mockConfig.get.withArgs('server.port').returns(3000);
 
+const mockKbnServer = {
+  status: {
+    toJSON: jest.fn().mockImplementation(() => ({
+      overall: { state: 'green' }
+    }))
+  }
+};
+
 describe('Metrics Collector', () => {
   describe('initialize', () => {
     it('should return stub metrics', () => {
       const collector = new MetricsCollector(mockServer, mockConfig);
-      expect(collector.getStats()).toMatchSnapshot();
+      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
     });
   });
 
@@ -88,7 +97,7 @@ describe('Metrics Collector', () => {
         psdelay: 0.33843398094177246,
         host: 'spicy.local',
       });
-      expect(collector.getStats()).toMatchSnapshot();
+      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
     });
 
     it('should accumulate counter metrics', async () => {
@@ -114,7 +123,7 @@ describe('Metrics Collector', () => {
         psdelay: 0.3764979839324951,
         host: 'spicy.local',
       });
-      expect(collector.getStats()).toMatchSnapshot();
+      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
 
       await collector.collect({
         requests: {
@@ -136,7 +145,7 @@ describe('Metrics Collector', () => {
         psdelay: 0.3764979839324951,
         host: 'spicy.local',
       });
-      expect(collector.getStats()).toMatchSnapshot();
+      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
 
       await collector.collect({
         requests: {
@@ -158,7 +167,7 @@ describe('Metrics Collector', () => {
         psdelay: 0.3764979839324951,
         host: 'spicy.local',
       });
-      expect(collector.getStats()).toMatchSnapshot();
+      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
     });
   });
 });

From 3478ed4f196b9e74ac8cfc0c8f32caea50c14c07 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Wed, 11 Jul 2018 14:48:47 -0700
Subject: [PATCH 03/42] fix api integration test

---
 test/api_integration/apis/stats/stats.js | 132 ++++++++++++-----------
 1 file changed, 69 insertions(+), 63 deletions(-)

diff --git a/test/api_integration/apis/stats/stats.js b/test/api_integration/apis/stats/stats.js
index 6e9bbb801482f..2c9c056884322 100644
--- a/test/api_integration/apis/stats/stats.js
+++ b/test/api_integration/apis/stats/stats.js
@@ -20,85 +20,91 @@
 import expect from 'expect.js';
 
 const assertStatsAndMetrics = body => {
-  expect(body.status.overall.state).to.be('green');
-  expect(body.status.statuses).to.be.an('array');
-  const kibanaPlugin = body.status.statuses.find(s => {
-    return s.id.indexOf('plugin:kibana') === 0;
-  });
-  expect(kibanaPlugin.state).to.be('green');
-
-  expect(body.name).to.be.a('string');
-  expect(body.uuid).to.be.a('string');
-
-  expect(body.version.number).to.be.a('string');
-
-  expect(body.process.mem.external_in_bytes).to.be.an('number');
-  expect(body.process.mem.heap_max_in_bytes).to.be.an('number');
-  expect(body.process.mem.heap_used_in_bytes).to.be.an('number');
-  expect(body.process.mem.resident_set_size_in_bytes).to.be.an('number');
-  expect(body.process.pid).to.be.an('number');
-  expect(body.process.uptime_ms).to.be.an('number');
+  expect(body.kibana.name).to.be.a('string');
+  expect(body.kibana.host).to.be.a('string');
+  expect(body.kibana.uuid).to.be.a('string');
+  expect(body.kibana.transport_address).to.be.a('string');
+  expect(body.kibana.version).to.be.a('string');
+  expect(body.kibana.snapshot).to.be.a('boolean');
+  expect(body.kibana.status).to.be('green');
+
+  expect(body.process.mem.external_bytes).to.be.a('number');
+  expect(body.process.mem.heap_max_bytes).to.be.a('number');
+  expect(body.process.mem.heap_used_bytes).to.be.a('number');
+  expect(body.process.mem.resident_set_size_bytes).to.be.a('number');
+  expect(body.process.pid).to.be.a('number');
+  expect(body.process.uptime_ms).to.be.a('number');
 
   expect(body.os.cpu.load_average['1m']).to.be.a('number');
+  expect(body.os.mem.free_bytes).to.be.a('number');
+  expect(body.os.mem.total_bytes).to.be.a('number');
+  expect(body.os.uptime_ms).to.be.a('number');
 
-  expect(body.response_times.avg_in_millis).not.to.be(null); // ok if is undefined
-  expect(body.response_times.max_in_millis).not.to.be(null); // ok if is undefined
+  expect(body.response_times.avg_ms).not.to.be(null); // ok if is undefined
+  expect(body.response_times.max_ms).not.to.be(null); // ok if is undefined
 
   expect(body.requests.status_codes).to.be.an('object');
+  expect(body.requests.total).to.be.a('number');
+  expect(body.requests.disconnects).to.be.a('number');
 
-  expect(body.sockets.http).to.be.an('object');
-  expect(body.sockets.https).to.be.an('object');
+  expect(body.sockets.http.total).to.be.a('number');
+  expect(body.sockets.https.total).to.be.a('number');
 
   expect(body.concurrent_connections).to.be.a('number');
-
-  expect(body.event_loop_delay).to.be.an('number');
+  expect(body.event_loop_delay).to.be.a('number');
 };
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
 
   describe('kibana stats api', () => {
-    it('should return the stats and metric fields without cluster_uuid when extended param is not present', () => {
-      return supertest
-        .get('/api/stats')
-        .expect('Content-Type', /json/)
-        .expect(200)
-        .then(({ body }) => {
-          expect(body.cluster_uuid).to.be(undefined);
-          assertStatsAndMetrics(body);
-        });
-    });
-    it('should return the stats and metric fields without cluster_uuid when extended param is given as false', () => {
-      return supertest
-        .get('/api/stats?extended=false')
-        .expect('Content-Type', /json/)
-        .expect(200)
-        .then(({ body }) => {
-          expect(body.cluster_uuid).to.be(undefined);
-          assertStatsAndMetrics(body);
-        });
-    });
-
-    it('should return the stats and metric fields with cluster_uuid when extended param is present', () => {
-      return supertest
-        .get('/api/stats?extended')
-        .expect('Content-Type', /json/)
-        .expect(200)
-        .then(({ body }) => {
-          expect(body.cluster_uuid).to.be.a('string');
-          assertStatsAndMetrics(body);
-        });
+    describe('basic', () => {
+      it('should return the stats and metric fields without cluster_uuid when extended param is not present', () => {
+        return supertest
+          .get('/api/stats')
+          .expect('Content-Type', /json/)
+          .expect(200)
+          .then(({ body }) => {
+            expect(body.cluster_uuid).to.be(undefined);
+            assertStatsAndMetrics(body);
+          });
+      });
+      it('should return the stats and metric fields without cluster_uuid when extended param is given as false', () => {
+        return supertest
+          .get('/api/stats?extended=false')
+          .expect('Content-Type', /json/)
+          .expect(200)
+          .then(({ body }) => {
+            expect(body.cluster_uuid).to.be(undefined);
+            assertStatsAndMetrics(body);
+          });
+      });
     });
 
-    it('should return the stats and metric fields with cluster_uuid when extended param is given as true', () => {
-      return supertest
-        .get('/api/stats?extended=true')
-        .expect('Content-Type', /json/)
-        .expect(200)
-        .then(({ body }) => {
-          expect(body.cluster_uuid).to.be.a('string');
-          assertStatsAndMetrics(body);
-        });
+    describe('extended', () => {
+      it('should return the stats and metric fields with cluster_uuid when extended param is present', () => {
+        return supertest
+          .get('/api/stats?extended')
+          .expect('Content-Type', /json/)
+          .expect(200)
+          .then(({ body }) => {
+            expect(body.cluster_uuid).to.be.a('string');
+            expect(body.usage.kibana).to.be.an('object');
+            assertStatsAndMetrics(body);
+          });
+      });
+
+      it('should return the stats and metric fields with cluster_uuid when extended param is given as true', () => {
+        return supertest
+          .get('/api/stats?extended=true')
+          .expect('Content-Type', /json/)
+          .expect(200)
+          .then(({ body }) => {
+            expect(body.cluster_uuid).to.be.a('string');
+            expect(body.usage.kibana).to.be.an('object');
+            assertStatsAndMetrics(body);
+          });
+      });
     });
   });
 }

From 12718bcd761785ae923efd526a9f149129ec22c6 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Wed, 11 Jul 2018 16:57:22 -0700
Subject: [PATCH 04/42] trash the original metrics collector

- constantly accumulating stats over time does not align with the existing behavior, which is to reset the stats to 0 whenever they are pulled
---
 src/server/status/index.js                    |   7 +-
 .../__mocks__/_fs_stubs.js                    |   0
 .../{metrics_collector => lib}/cgroup.js      |   0
 .../{metrics_collector => lib}/cgroup.test.js |   0
 .../{metrics_collector => lib}/metrics.js     |   0
 .../metrics.test.js                           |   0
 .../__snapshots__/metrics.test.js.snap        | 114 --------
 .../metrics_collector.test.js.snap            | 274 ------------------
 src/server/status/metrics_collector/index.js  |  20 --
 .../metrics_collector/metrics_collector.js    | 118 --------
 .../metrics_collector.test.js                 | 173 -----------
 .../metrics_collector/sum_accumulate.test.js  | 101 -------
 12 files changed, 2 insertions(+), 805 deletions(-)
 rename src/server/status/{metrics_collector => lib}/__mocks__/_fs_stubs.js (100%)
 rename src/server/status/{metrics_collector => lib}/cgroup.js (100%)
 rename src/server/status/{metrics_collector => lib}/cgroup.test.js (100%)
 rename src/server/status/{metrics_collector => lib}/metrics.js (100%)
 rename src/server/status/{metrics_collector => lib}/metrics.test.js (100%)
 delete mode 100644 src/server/status/metrics_collector/__snapshots__/metrics.test.js.snap
 delete mode 100644 src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap
 delete mode 100644 src/server/status/metrics_collector/index.js
 delete mode 100644 src/server/status/metrics_collector/metrics_collector.js
 delete mode 100644 src/server/status/metrics_collector/metrics_collector.test.js
 delete mode 100644 src/server/status/metrics_collector/sum_accumulate.test.js

diff --git a/src/server/status/index.js b/src/server/status/index.js
index 6dc45655fd3e5..685bd3a4c095b 100644
--- a/src/server/status/index.js
+++ b/src/server/status/index.js
@@ -18,12 +18,10 @@
  */
 
 import ServerStatus from './server_status';
-import { MetricsCollector } from './metrics_collector';
-import { Metrics } from './metrics_collector/metrics';
+import { Metrics } from './lib/metrics';
 import { registerStatusPage, registerStatusApi, registerStatsApi } from './routes';
 
 export function statusMixin(kbnServer, server, config) {
-  const collector = new MetricsCollector(server, config);
   kbnServer.status = new ServerStatus(kbnServer.server);
 
   const { ['even-better']: evenBetter } = server.plugins;
@@ -33,12 +31,11 @@ export function statusMixin(kbnServer, server, config) {
 
     evenBetter.monitor.on('ops', event => {
       metrics.capture(event).then(data => { kbnServer.metrics = data; }); // for status API (to deprecate in next major)
-      collector.collect(event); // for metrics API (replacement API)
     });
   }
 
   // init routes
   registerStatusPage(kbnServer, server, config);
   registerStatusApi(kbnServer, server, config);
-  registerStatsApi(kbnServer, server, config, collector);
+  registerStatsApi(kbnServer, server, config);
 }
diff --git a/src/server/status/metrics_collector/__mocks__/_fs_stubs.js b/src/server/status/lib/__mocks__/_fs_stubs.js
similarity index 100%
rename from src/server/status/metrics_collector/__mocks__/_fs_stubs.js
rename to src/server/status/lib/__mocks__/_fs_stubs.js
diff --git a/src/server/status/metrics_collector/cgroup.js b/src/server/status/lib/cgroup.js
similarity index 100%
rename from src/server/status/metrics_collector/cgroup.js
rename to src/server/status/lib/cgroup.js
diff --git a/src/server/status/metrics_collector/cgroup.test.js b/src/server/status/lib/cgroup.test.js
similarity index 100%
rename from src/server/status/metrics_collector/cgroup.test.js
rename to src/server/status/lib/cgroup.test.js
diff --git a/src/server/status/metrics_collector/metrics.js b/src/server/status/lib/metrics.js
similarity index 100%
rename from src/server/status/metrics_collector/metrics.js
rename to src/server/status/lib/metrics.js
diff --git a/src/server/status/metrics_collector/metrics.test.js b/src/server/status/lib/metrics.test.js
similarity index 100%
rename from src/server/status/metrics_collector/metrics.test.js
rename to src/server/status/lib/metrics.test.js
diff --git a/src/server/status/metrics_collector/__snapshots__/metrics.test.js.snap b/src/server/status/metrics_collector/__snapshots__/metrics.test.js.snap
deleted file mode 100644
index 72f148f69dbe6..0000000000000
--- a/src/server/status/metrics_collector/__snapshots__/metrics.test.js.snap
+++ /dev/null
@@ -1,114 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`Metrics capture merges all metrics 1`] = `
-Object {
-  "a": Array [
-    Object {
-      "b": 2,
-      "c": 3,
-    },
-    Object {
-      "d": 4,
-      "e": 5,
-    },
-  ],
-  "collection_interval_in_millis": 5000,
-  "last_updated": "2017-04-14T18:35:41.534Z",
-  "process": Object {
-    "uptime_ms": 1980,
-  },
-}
-`;
-
-exports[`Metrics captureEvent parses event with missing fields / NaN for responseTimes.avg 1`] = `
-Object {
-  "concurrent_connections": undefined,
-  "event_loop_delay": undefined,
-  "os": Object {
-    "cpu": Object {
-      "load_average": Object {
-        "15m": undefined,
-        "1m": undefined,
-        "5m": undefined,
-      },
-    },
-    "mem": Object {
-      "free_in_bytes": 12,
-      "total_in_bytes": 24,
-    },
-    "uptime_ms": 12000000,
-  },
-  "process": Object {
-    "mem": Object {
-      "external_in_bytes": undefined,
-      "heap_max_in_bytes": undefined,
-      "heap_used_in_bytes": undefined,
-      "resident_set_size_in_bytes": undefined,
-    },
-    "pid": 8675309,
-    "uptime_ms": 5000000,
-  },
-  "requests": Object {
-    "disconnects": 0,
-    "status_codes": Object {
-      "200": 22,
-    },
-    "total": 22,
-  },
-  "response_times": Object {
-    "avg_in_millis": undefined,
-    "max_in_millis": 4,
-  },
-  "sockets": undefined,
-}
-`;
-
-exports[`Metrics captureEvent parses the hapi event 1`] = `
-Object {
-  "concurrent_connections": 0,
-  "event_loop_delay": 1.6091690063476562,
-  "os": Object {
-    "cpu": Object {
-      "load_average": Object {
-        "15m": 1.89794921875,
-        "1m": 2.20751953125,
-        "5m": 2.02294921875,
-      },
-    },
-    "mem": Object {
-      "free_in_bytes": 12,
-      "total_in_bytes": 24,
-    },
-    "uptime_ms": 12000000,
-  },
-  "process": Object {
-    "mem": Object {
-      "external_in_bytes": 1779619,
-      "heap_max_in_bytes": 168194048,
-      "heap_used_in_bytes": 130553400,
-      "resident_set_size_in_bytes": 193716224,
-    },
-    "pid": 8675309,
-    "uptime_ms": 5000000,
-  },
-  "requests": Object {
-    "disconnects": 0,
-    "status_codes": Object {
-      "200": 22,
-    },
-    "total": 22,
-  },
-  "response_times": Object {
-    "avg_in_millis": 1.8636363636363635,
-    "max_in_millis": 4,
-  },
-  "sockets": Object {
-    "http": Object {
-      "total": 0,
-    },
-    "https": Object {
-      "total": 0,
-    },
-  },
-}
-`;
diff --git a/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap b/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap
deleted file mode 100644
index 014876004c3a7..0000000000000
--- a/src/server/status/metrics_collector/__snapshots__/metrics_collector.test.js.snap
+++ /dev/null
@@ -1,274 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`Metrics Collector collection should accumulate counter metrics 1`] = `
-Object {
-  "collection_interval_in_millis": "test-123",
-  "concurrent_connections": 0,
-  "event_loop_delay": 0.3764979839324951,
-  "kibana": Object {
-    "host": "test-123",
-    "name": "test-123",
-    "snapshot": false,
-    "status": "green",
-    "transport_address": "test-123:3000",
-    "uuid": "test-123",
-    "version": "test-123",
-  },
-  "last_updated": "2018-04-19T21:50:54.366Z",
-  "os": Object {
-    "cpu": Object {
-      "load_average": Object {
-        "15m": 1.81201171875,
-        "1m": 1.97119140625,
-        "5m": 1.90283203125,
-      },
-    },
-    "mem": Object {
-      "free_in_bytes": 12,
-      "total_in_bytes": 24,
-    },
-    "uptime_ms": NaN,
-  },
-  "process": Object {
-    "mem": Object {
-      "external_in_bytes": 25028,
-      "heap_max_in_bytes": 15548416,
-      "heap_used_in_bytes": 12996392,
-      "resident_set_size_in_bytes": 36085760,
-    },
-    "pid": 7777,
-    "uptime_ms": 6666000,
-  },
-  "requests": Object {
-    "disconnects": 0,
-    "status_codes": Object {
-      "200": 8,
-    },
-    "total": 8,
-  },
-  "response_times": Object {
-    "avg_in_millis": 19,
-    "max_in_millis": 19,
-  },
-  "sockets": Object {
-    "http": Object {
-      "total": 0,
-    },
-    "https": Object {
-      "total": 0,
-    },
-  },
-}
-`;
-
-exports[`Metrics Collector collection should accumulate counter metrics 2`] = `
-Object {
-  "collection_interval_in_millis": "test-123",
-  "concurrent_connections": 0,
-  "event_loop_delay": 0.7529959678649902,
-  "kibana": Object {
-    "host": "test-123",
-    "name": "test-123",
-    "snapshot": false,
-    "status": "green",
-    "transport_address": "test-123:3000",
-    "uuid": "test-123",
-    "version": "test-123",
-  },
-  "last_updated": "2018-04-19T21:50:54.366Z",
-  "os": Object {
-    "cpu": Object {
-      "load_average": Object {
-        "15m": 1.81201171875,
-        "1m": 1.97119140625,
-        "5m": 1.90283203125,
-      },
-    },
-    "mem": Object {
-      "free_in_bytes": 12,
-      "total_in_bytes": 24,
-    },
-    "uptime_ms": NaN,
-  },
-  "process": Object {
-    "mem": Object {
-      "external_in_bytes": 25028,
-      "heap_max_in_bytes": 15548416,
-      "heap_used_in_bytes": 12996392,
-      "resident_set_size_in_bytes": 36085760,
-    },
-    "pid": 7777,
-    "uptime_ms": 6666000,
-  },
-  "requests": Object {
-    "disconnects": 0,
-    "status_codes": Object {
-      "200": 16,
-    },
-    "total": 16,
-  },
-  "response_times": Object {
-    "avg_in_millis": 38,
-    "max_in_millis": 38,
-  },
-  "sockets": Object {
-    "http": Object {
-      "total": 0,
-    },
-    "https": Object {
-      "total": 0,
-    },
-  },
-}
-`;
-
-exports[`Metrics Collector collection should accumulate counter metrics 3`] = `
-Object {
-  "collection_interval_in_millis": "test-123",
-  "concurrent_connections": 0,
-  "event_loop_delay": 1.1294939517974854,
-  "kibana": Object {
-    "host": "test-123",
-    "name": "test-123",
-    "snapshot": false,
-    "status": "green",
-    "transport_address": "test-123:3000",
-    "uuid": "test-123",
-    "version": "test-123",
-  },
-  "last_updated": "2018-04-19T21:50:54.366Z",
-  "os": Object {
-    "cpu": Object {
-      "load_average": Object {
-        "15m": 1.81201171875,
-        "1m": 1.97119140625,
-        "5m": 1.90283203125,
-      },
-    },
-    "mem": Object {
-      "free_in_bytes": 12,
-      "total_in_bytes": 24,
-    },
-    "uptime_ms": NaN,
-  },
-  "process": Object {
-    "mem": Object {
-      "external_in_bytes": 25028,
-      "heap_max_in_bytes": 15548416,
-      "heap_used_in_bytes": 12996392,
-      "resident_set_size_in_bytes": 36085760,
-    },
-    "pid": 7777,
-    "uptime_ms": 6666000,
-  },
-  "requests": Object {
-    "disconnects": 0,
-    "status_codes": Object {
-      "200": 24,
-    },
-    "total": 24,
-  },
-  "response_times": Object {
-    "avg_in_millis": 57,
-    "max_in_millis": 57,
-  },
-  "sockets": Object {
-    "http": Object {
-      "total": 0,
-    },
-    "https": Object {
-      "total": 0,
-    },
-  },
-}
-`;
-
-exports[`Metrics Collector collection should update stats with new data 1`] = `
-Object {
-  "collection_interval_in_millis": "test-123",
-  "concurrent_connections": 0,
-  "event_loop_delay": 0.33843398094177246,
-  "kibana": Object {
-    "host": "test-123",
-    "name": "test-123",
-    "snapshot": false,
-    "status": "green",
-    "transport_address": "test-123:3000",
-    "uuid": "test-123",
-    "version": "test-123",
-  },
-  "last_updated": "2018-04-19T21:50:54.366Z",
-  "os": Object {
-    "cpu": Object {
-      "load_average": Object {
-        "15m": 1.8154296875,
-        "1m": 1.68017578125,
-        "5m": 1.7685546875,
-      },
-    },
-    "mem": Object {
-      "free_in_bytes": 12,
-      "total_in_bytes": 24,
-    },
-    "uptime_ms": NaN,
-  },
-  "process": Object {
-    "mem": Object {
-      "external_in_bytes": 25028,
-      "heap_max_in_bytes": 15548416,
-      "heap_used_in_bytes": 12911128,
-      "resident_set_size_in_bytes": 35307520,
-    },
-    "pid": 7777,
-    "uptime_ms": 6666000,
-  },
-  "requests": Object {
-    "disconnects": 0,
-    "status_codes": Object {
-      "200": 4,
-    },
-    "total": 4,
-  },
-  "response_times": Object {
-    "avg_in_millis": 13,
-    "max_in_millis": 13,
-  },
-  "sockets": Object {
-    "http": Object {
-      "total": 0,
-    },
-    "https": Object {
-      "total": 0,
-    },
-  },
-}
-`;
-
-exports[`Metrics Collector initialize should return stub metrics 1`] = `
-Object {
-  "kibana": Object {
-    "host": "test-123",
-    "name": "test-123",
-    "snapshot": false,
-    "status": "green",
-    "transport_address": "test-123:3000",
-    "uuid": "test-123",
-    "version": "test-123",
-  },
-  "os": Object {
-    "cpu": Object {},
-    "mem": Object {},
-  },
-  "process": Object {
-    "mem": Object {},
-  },
-  "requests": Object {
-    "status_codes": Object {},
-  },
-  "response_times": Object {},
-  "sockets": Object {
-    "http": Object {},
-    "https": Object {},
-  },
-}
-`;
diff --git a/src/server/status/metrics_collector/index.js b/src/server/status/metrics_collector/index.js
deleted file mode 100644
index ebdecd61942ef..0000000000000
--- a/src/server/status/metrics_collector/index.js
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-export { MetricsCollector } from './metrics_collector';
diff --git a/src/server/status/metrics_collector/metrics_collector.js b/src/server/status/metrics_collector/metrics_collector.js
deleted file mode 100644
index 3c1b3be7de914..0000000000000
--- a/src/server/status/metrics_collector/metrics_collector.js
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { Metrics } from './metrics';
-
-const matchSnapshot = /-SNAPSHOT$/;
-
-/*
- * Persist operational data for machine reading
- * sets the latest gauge values
- * sums the latest accumulative values
- */
-export class MetricsCollector {
-  constructor(server, config) {
-
-    // NOTE we need access to config every time this is used because uuid is
-    // managed by the kibana core_plugin, which is initialized AFTER kbn_server
-    this._getKibanaStats = () => {
-      return {
-        name: config.get('server.name'),
-        host: config.get('server.host'),
-        uuid: config.get('server.uuid'),
-        transport_address: `${config.get('server.host')}:${config.get('server.port')}`,
-        version: config.get('pkg.version'),
-        snapshot: matchSnapshot.test(config.get('pkg.version')),
-      };
-    };
-
-    this._stats = Metrics.getStubMetrics();
-    this._metrics = new Metrics(config, server); // TODO: deprecate status API that uses Metrics class, move it this module, fix the order of its constructor params
-  }
-
-  /*
-   * Accumulate metrics by summing values in an accumulutor object with the next values
-   *
-   * @param {String} property The property of the objects to roll up
-   * @param {Object} accum The accumulator object
-   * @param {Object} next The object containing next values
-   */
-  static sumAccumulate(property, accum, next) {
-    const accumValue = accum[property];
-    const nextValue = next[property];
-
-    if (nextValue === null || nextValue === undefined) {
-      return; // do not accumulate null/undefined since it can't be part of a sum
-    } else if (nextValue.constructor === Object) { // nested structure, object
-      const newProps = {};
-      for (const innerKey in nextValue) {
-        if (nextValue.hasOwnProperty(innerKey)) {
-          const tempAccumValue = accumValue || {};
-          newProps[innerKey] = MetricsCollector.sumAccumulate(innerKey, tempAccumValue, nextValue);
-        }
-      }
-      return { // merge the newly summed nested values
-        ...accumValue,
-        ...newProps,
-      };
-    } else if (nextValue.constructor === Number) {
-      // leaf value
-      if (nextValue || nextValue === 0) {
-        const tempAccumValue = accumValue || 0; // treat null / undefined as 0
-        const tempNextValue = nextValue || 0;
-        return tempAccumValue + tempNextValue; // perform sum
-      }
-    } else {
-      return; // drop unknown type
-    }
-  }
-
-  async collect(event) {
-    const capturedEvent = await this._metrics.capture(event); // wait for cgroup measurement
-    const { process, os, ...metrics } = capturedEvent;
-
-    const stats = {
-      // gauge values
-      ...metrics,
-      process,
-      os,
-
-      // accumulative counters
-      response_times: MetricsCollector.sumAccumulate('response_times', this._stats, metrics),
-      requests: MetricsCollector.sumAccumulate('requests', this._stats, metrics),
-      concurrent_connections: MetricsCollector.sumAccumulate('concurrent_connections', this._stats, metrics),
-      sockets: MetricsCollector.sumAccumulate('sockets', this._stats, metrics),
-      event_loop_delay: MetricsCollector.sumAccumulate('event_loop_delay', this._stats, metrics),
-    };
-
-    this._stats = stats;
-    return stats;
-  }
-
-  getStats(kbnServer) {
-    const status = kbnServer.status.toJSON();
-    return {
-      kibana: {
-        ...this._getKibanaStats(),
-        status: status.overall.state
-      },
-      ...this._stats
-    };
-  }
-}
diff --git a/src/server/status/metrics_collector/metrics_collector.test.js b/src/server/status/metrics_collector/metrics_collector.test.js
deleted file mode 100644
index fbecb65dee61c..0000000000000
--- a/src/server/status/metrics_collector/metrics_collector.test.js
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-jest.mock('os', () => ({
-  freemem: jest.fn(),
-  totalmem: jest.fn(),
-  uptime: jest.fn()
-}));
-
-const mockProcessUptime = jest.fn().mockImplementation(() => 6666);
-jest.mock('process', () => ({
-  uptime: mockProcessUptime
-}));
-
-import os from 'os';
-import sinon from 'sinon';
-import { MetricsCollector } from './';
-
-const mockServer = {};
-const mockConfig = {
-  get: sinon.stub(),
-};
-mockConfig.get.returns('test-123');
-mockConfig.get.withArgs('server.port').returns(3000);
-
-const mockKbnServer = {
-  status: {
-    toJSON: jest.fn().mockImplementation(() => ({
-      overall: { state: 'green' }
-    }))
-  }
-};
-
-describe('Metrics Collector', () => {
-  describe('initialize', () => {
-    it('should return stub metrics', () => {
-      const collector = new MetricsCollector(mockServer, mockConfig);
-      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
-    });
-  });
-
-  describe('collection', () => {
-    os.freemem.mockImplementation(() => 12);
-    os.totalmem.mockImplementation(() => 24);
-
-    Object.defineProperty(process, 'pid', { value: 7777 });
-    Object.defineProperty(process, 'uptime', { value: mockProcessUptime });
-
-    let sandbox;
-    let clock;
-    beforeAll(() => {
-      sandbox = sinon.createSandbox();
-      clock = sandbox.useFakeTimers(1524174654366);
-    });
-
-    afterAll(() => {
-      clock.restore();
-      sandbox.restore();
-    });
-
-    it('should update stats with new data', async () => {
-      const collector = new MetricsCollector(mockServer, mockConfig);
-
-      await collector.collect({
-        requests: {
-          '3000': { total: 4, disconnects: 0, statusCodes: { '200': 4 } },
-        },
-        responseTimes: { '3000': { avg: 13, max: 13 } },
-        sockets: { http: { total: 0 }, https: { total: 0 } },
-        osload: [1.68017578125, 1.7685546875, 1.8154296875],
-        osmem: { total: 17179869184, free: 3984404480 },
-        psmem: {
-          rss: 35307520,
-          heapTotal: 15548416,
-          heapUsed: 12911128,
-          external: 25028,
-        },
-        concurrents: { '3000': 0 },
-        osup: 965002,
-        psup: 29.466,
-        psdelay: 0.33843398094177246,
-        host: 'spicy.local',
-      });
-      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
-    });
-
-    it('should accumulate counter metrics', async () => {
-      const collector = new MetricsCollector(mockServer, mockConfig);
-
-      await collector.collect({
-        requests: {
-          '3000': { total: 8, disconnects: 0, statusCodes: { '200': 8 } },
-        },
-        responseTimes: { '3000': { avg: 19, max: 19 } },
-        sockets: { http: { total: 0 }, https: { total: 0 } },
-        osload: [1.97119140625, 1.90283203125, 1.81201171875],
-        osmem: { total: 17179869184, free: 3987533824 },
-        psmem: {
-          rss: 36085760,
-          heapTotal: 15548416,
-          heapUsed: 12996392,
-          external: 25028,
-        },
-        concurrents: { '3000': 0 },
-        osup: 965606,
-        psup: 22.29,
-        psdelay: 0.3764979839324951,
-        host: 'spicy.local',
-      });
-      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
-
-      await collector.collect({
-        requests: {
-          '3000': { total: 8, disconnects: 0, statusCodes: { '200': 8 } },
-        },
-        responseTimes: { '3000': { avg: 19, max: 19 } },
-        sockets: { http: { total: 0 }, https: { total: 0 } },
-        osload: [1.97119140625, 1.90283203125, 1.81201171875],
-        osmem: { total: 17179869184, free: 3987533824 },
-        psmem: {
-          rss: 36085760,
-          heapTotal: 15548416,
-          heapUsed: 12996392,
-          external: 25028,
-        },
-        concurrents: { '3000': 0 },
-        osup: 965606,
-        psup: 22.29,
-        psdelay: 0.3764979839324951,
-        host: 'spicy.local',
-      });
-      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
-
-      await collector.collect({
-        requests: {
-          '3000': { total: 8, disconnects: 0, statusCodes: { '200': 8 } },
-        },
-        responseTimes: { '3000': { avg: 19, max: 19 } },
-        sockets: { http: { total: 0 }, https: { total: 0 } },
-        osload: [1.97119140625, 1.90283203125, 1.81201171875],
-        osmem: { total: 17179869184, free: 3987533824 },
-        psmem: {
-          rss: 36085760,
-          heapTotal: 15548416,
-          heapUsed: 12996392,
-          external: 25028,
-        },
-        concurrents: { '3000': 0 },
-        osup: 965606,
-        psup: 22.29,
-        psdelay: 0.3764979839324951,
-        host: 'spicy.local',
-      });
-      expect(collector.getStats(mockKbnServer)).toMatchSnapshot();
-    });
-  });
-});
diff --git a/src/server/status/metrics_collector/sum_accumulate.test.js b/src/server/status/metrics_collector/sum_accumulate.test.js
deleted file mode 100644
index 660a1d015a873..0000000000000
--- a/src/server/status/metrics_collector/sum_accumulate.test.js
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { MetricsCollector } from './';
-
-const { sumAccumulate } = MetricsCollector;
-
-describe('Accumulate By Summing Metrics', function () {
-  it('should accumulate empty object with nothing as nothing', () => {
-    const accum = { blues: {} };
-    const current = sumAccumulate('blues', accum, {});
-    expect(current).toEqual(undefined);
-  });
-
-  it('should return data to merge with initial empty data', () => {
-    let accum = { blues: {} };
-    const next = { blues: { total: 1 } };
-    const accumulated = sumAccumulate('blues', accum, next);
-    accum = { ...accum, blues: accumulated };
-    expect(accum).toEqual({ blues: { total: 1 } });
-  });
-
-  it('should return data to merge with already accumulated data', () => {
-    let currentProp;
-    let accumulated;
-
-    // initial
-    let accum = {
-      reds: 1,
-      oranges: { total: 2 },
-      yellows: { total: 3 },
-      greens: { total: 4 },
-      blues: { dislikes: 2, likes: 3, total: 5 },
-      indigos: { total: 6 },
-      violets: { total: 7 },
-    };
-
-    // first accumulation - existing nested object
-    currentProp = 'blues';
-    accumulated = sumAccumulate(currentProp, accum, {
-      [currentProp]: { likes: 2, total: 2 },
-    });
-    accum = { ...accum, [currentProp]: accumulated };
-    expect(accum).toEqual({
-      reds: 1,
-      oranges: { total: 2 },
-      yellows: { total: 3 },
-      greens: { total: 4 },
-      blues: { dislikes: 2, likes: 5, total: 7 },
-      indigos: { total: 6 },
-      violets: { total: 7 },
-    });
-
-    // second accumulation - existing non-nested object
-    currentProp = 'reds';
-    accumulated = sumAccumulate(currentProp, accum, { [currentProp]: 2 });
-    accum = { ...accum, [currentProp]: accumulated };
-    expect(accum).toEqual({
-      reds: 3,
-      oranges: { total: 2 },
-      yellows: { total: 3 },
-      greens: { total: 4 },
-      blues: { dislikes: 2, likes: 5, total: 7 },
-      indigos: { total: 6 },
-      violets: { total: 7 },
-    });
-
-    // third accumulation - new nested object prop
-    currentProp = 'ultraviolets';
-    accumulated = sumAccumulate(currentProp, accum, {
-      [currentProp]: { total: 1, likes: 1, dislikes: 0 },
-    });
-    accum = { ...accum, [currentProp]: accumulated };
-    expect(accum).toEqual({
-      reds: 3,
-      oranges: { total: 2 },
-      yellows: { total: 3 },
-      greens: { total: 4 },
-      blues: { dislikes: 2, likes: 5, total: 7 },
-      indigos: { total: 6 },
-      violets: { total: 7 },
-      ultraviolets: { dislikes: 0, likes: 1, total: 1 },
-    });
-  });
-});

From d59cd4d426fbfbb091fb7d99909bdd0047618444 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Wed, 11 Jul 2018 16:58:30 -0700
Subject: [PATCH 05/42] move some logic out of the collector types combiner
 into inline

- change the signature of sourceKibana
---
 .../status/routes/api/register_stats.js       | 13 ++++++++-----
 src/server/usage/classes/collector_set.js     |  4 ++++
 x-pack/plugins/monitoring/init.js             |  7 ++++---
 .../server/kibana_monitoring/bulk_uploader.js |  2 +-
 .../collectors/get_ops_stats_collector.js     | 10 ++++++++--
 .../collectors/get_settings_collector.js      |  8 ++++++--
 .../lib/get_collector_types_combiner.js       | 19 ++++---------------
 .../server/kibana_monitoring/lib/index.js     |  1 +
 .../kibana_monitoring/lib/source_kibana.js    |  5 +++--
 9 files changed, 39 insertions(+), 30 deletions(-)

diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js
index 0548b53d3e0ce..4a17245a210fb 100644
--- a/src/server/status/routes/api/register_stats.js
+++ b/src/server/status/routes/api/register_stats.js
@@ -21,7 +21,7 @@ import Joi from 'joi';
 import { wrapAuthConfig } from '../../wrap_auth_config';
 import { setApiFieldNames } from '../../lib';
 
-async function getExtended(req, server) {
+async function getExtended(req, server, collectorSet) {
   const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('admin'); // admin cluster, get info on internal system
   const callCluster = (...args) => callWithRequest(req, ...args);
 
@@ -35,7 +35,6 @@ async function getExtended(req, server) {
 
   let usage;
   try {
-    const { collectorSet } = server.usage;
     const usageRaw = await collectorSet.bulkFetchUsage(callCluster);
     usage = collectorSet.summarizeStats(usageRaw);
   } catch (err) {
@@ -54,8 +53,11 @@ async function getExtended(req, server) {
  * - No value or 'false' is isExtended = false
  * - Any other value causes a statusCode 400 response (Bad Request)
  */
-export function registerStatsApi(kbnServer, server, config, collector) {
+export function registerStatsApi(kbnServer, server, config) {
   const wrapAuth = wrapAuthConfig(config.get('status.allowAnonymous'));
+
+  const { collectorSet } = server.usage;
+
   server.route(
     wrapAuth({
       method: 'GET',
@@ -75,11 +77,12 @@ export function registerStatsApi(kbnServer, server, config, collector) {
         let clusterUuid;
         let usage;
         if (isExtended) {
-          ({ clusterUuid, usage } = await getExtended(req, server));
+          ({ clusterUuid, usage } = await getExtended(req, server, collectorSet));
         }
 
+        const kibanaCollector = collectorSet.getCollectorByType('kibana_stats');
         const stats = setApiFieldNames({
-          ...collector.getStats(kbnServer),
+          ...kibanaCollector.fetch(),
           cluster_uuid: clusterUuid,
           usage,
         });
diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js
index 53ae4c77ea0d8..0790c5b56f173 100644
--- a/src/server/usage/classes/collector_set.js
+++ b/src/server/usage/classes/collector_set.js
@@ -64,6 +64,10 @@ export class CollectorSet {
     }
   }
 
+  getCollectorByType(type) {
+    return this._collectors.find(c => c.type === type);
+  }
+
   /*
    * Call a bunch of fetch methods and then do them in bulk
    * @param {Array} collectors - an array of collectors, default to all registered collectors
diff --git a/x-pack/plugins/monitoring/init.js b/x-pack/plugins/monitoring/init.js
index f5d97ead958a7..1769cb7da4bb3 100644
--- a/x-pack/plugins/monitoring/init.js
+++ b/x-pack/plugins/monitoring/init.js
@@ -26,14 +26,15 @@ import {
  * @param server {Object} HapiJS server instance
  */
 export const init = (monitoringPlugin, server) => {
+  const kbnServer = monitoringPlugin.kbnServer;
   const config = server.config();
   const { collectorSet } = server.usage;
   /*
    * Register collector objects for stats to show up in the APIs
    */
-  collectorSet.register(getOpsStatsCollector(server));
+  collectorSet.register(getOpsStatsCollector(server, kbnServer)); // TODO move to OSS probably
+  collectorSet.register(getSettingsCollector(server, kbnServer));
   collectorSet.register(getKibanaUsageCollector(server));
-  collectorSet.register(getSettingsCollector(server));
 
   /*
    * Instantiate and start the internal background task that calls collector
@@ -53,7 +54,7 @@ export const init = (monitoringPlugin, server) => {
     }
   });
 
-  const bulkUploader = initBulkUploader(monitoringPlugin.kbnServer, server);
+  const bulkUploader = initBulkUploader(kbnServer, server);
   const kibanaCollectionEnabled = config.get('xpack.monitoring.kibana.collection.enabled');
   const { info: xpackMainInfo } = xpackMainPlugin;
 
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
index de8fa0969c719..bfd77b659b1fd 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
@@ -101,7 +101,7 @@ export class BulkUploader {
 
     if (payload.length > 0) {
       try {
-        const combinedData = this._combineTypes(payload); // use the collector types combiner
+        const combinedData = this._combineTypes(payload); // use the collector types combiner to combine the data into a single bulk upload object
         this._log.debug(`Uploading bulk stats payload to the local cluster`);
         this._onPayload(flatten(combinedData));
       } catch (err) {
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
index cd6af97a029b4..9e551e89480a0 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
@@ -6,11 +6,12 @@
 
 import { KIBANA_STATS_TYPE } from '../../../common/constants';
 import { opsBuffer } from './ops_buffer';
+import { sourceKibana } from '../lib';
 
 /*
  * Initialize a collector for Kibana Ops Stats
  */
-export function getOpsStatsCollector(server) {
+export function getOpsStatsCollector(server, kbnServer) {
   let monitor;
   const buffer = opsBuffer(server);
   const onOps = event => buffer.push(event);
@@ -46,6 +47,11 @@ export function getOpsStatsCollector(server) {
   return collectorSet.makeStatsCollector({
     type: KIBANA_STATS_TYPE,
     init: start,
-    fetch: buffer.flush
+    fetch: () => {
+      return {
+        kibana: sourceKibana(server, kbnServer),
+        ...buffer.flush()
+      };
+    }
   });
 }
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js
index 1a029169b365e..d9913b691e97f 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js
@@ -7,6 +7,7 @@
 import { get } from 'lodash';
 import { XPACK_DEFAULT_ADMIN_EMAIL_UI_SETTING } from '../../../../../server/lib/constants';
 import { KIBANA_SETTINGS_TYPE } from '../../../common/constants';
+import { sourceKibana } from '../lib';
 
 /*
  * Check if Cluster Alert email notifications is enabled in config
@@ -53,7 +54,7 @@ export async function checkForEmailValue(
   }
 }
 
-export function getSettingsCollector(server) {
+export function getSettingsCollector(server, kbnServer) {
   const config = server.config();
 
   const { collectorSet } = server.usage;
@@ -78,7 +79,10 @@ export function getSettingsCollector(server) {
       // remember the current email so that we can mark it as successful if the bulk does not error out
       shouldUseNull = !!defaultAdminEmail;
 
-      return kibanaSettingsData;
+      return {
+        kibana: sourceKibana(server, kbnServer),
+        ...kibanaSettingsData
+      };
     }
   });
 }
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js
index f50b080a9afc9..4e029fff0edfc 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js
@@ -4,14 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { get, set, omit } from 'lodash';
+import { get, set } from 'lodash';
 import {
   KIBANA_STATS_TYPE,
   KIBANA_SETTINGS_TYPE,
   KIBANA_USAGE_TYPE,
 } from '../../../common/constants';
 import { KIBANA_REPORTING_TYPE } from '../../../../reporting/common/constants';
-import { sourceKibana } from './source_kibana';
 
 /*
  * Combine stats collected from different sources into a single bulk payload.
@@ -32,7 +31,7 @@ import { sourceKibana } from './source_kibana';
  *  - Individual plugin usage stats can go into a new API similar to the `_xpack/usage` API in ES.
  *  - Each plugin will have its own top-level property in the responses for these APIs.
  */
-export function getCollectorTypesCombiner(kbnServer, config, _sourceKibana = sourceKibana) {
+export function getCollectorTypesCombiner() {
   return payload => {
     // default the item to [] to allow destructuring
     const findItem = type => payload.find(item => get(item, '[0].index._type') === type) || [];
@@ -42,20 +41,13 @@ export function getCollectorTypesCombiner(kbnServer, config, _sourceKibana = sou
     const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE);
     const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE);
 
-    // sourceKibana uses "host" from the kibana stats payload
-    const host = get(statsPayload, 'host');
-    const kibana = _sourceKibana(kbnServer, config, host);
-
     if (statsHeader && statsPayload) {
       const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE);
       const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null;
       const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null; // this is an abstraction leak
       statsResult = [
         statsHeader,
-        {
-          ...omit(statsPayload, 'host'), // remove the temp host field
-          kibana,
-        }
+        statsPayload
       ];
       if (kibanaUsage) {
         set(statsResult, '[1].usage', kibanaUsage);
@@ -71,10 +63,7 @@ export function getCollectorTypesCombiner(kbnServer, config, _sourceKibana = sou
     if (settingsHeader && settingsPayload) {
       settingsResult = [
         settingsHeader,
-        {
-          ...settingsPayload,
-          kibana
-        }
+        settingsPayload,
       ];
     }
 
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js
index 2b81cd36fe302..ea4fb99abf0ea 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js
@@ -7,3 +7,4 @@
 export { getCollectorTypesCombiner } from './get_collector_types_combiner';
 export { sendBulkPayload } from './send_bulk_payload';
 export { monitoringBulk } from './monitoring_bulk';
+export { sourceKibana } from './source_kibana';
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js
index 23f7c848d4bbd..b965c526d2323 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js
@@ -19,14 +19,15 @@ const snapshotRegex = /-snapshot/i;
  * @param {String} host Kibana host
  * @return {Object} The object containing a "kibana" field and source instance details.
  */
-export function sourceKibana(kbnServer, config, host) {
+export function sourceKibana(server, kbnServer) {
+  const config = server.config();
   const status = kbnServer.status.toJSON();
 
   return {
     uuid: config.get('server.uuid'),
     name: config.get('server.name'),
     index: config.get('kibana.index'),
-    host,
+    host: config.get('server.host'),
     transport_address: `${config.get('server.host')}:${config.get('server.port')}`,
     version: kbnServer.version.replace(snapshotRegex, ''),
     snapshot: snapshotRegex.test(kbnServer.version),

From 027ef25c4037fcf8099fa2ce49f33c0933c83328 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 10:27:09 -0700
Subject: [PATCH 06/42] Make a new stats collector for the API

- to not clear the data when pulling via the api
- fetching is a read-only thing
---
 .../collectors/get_ops_stats_collector.js     | 38 ++++++++++++++
 src/server/status/collectors/index.js         | 20 ++++++++
 src/server/status/constants.js                | 20 ++++++++
 src/server/status/index.js                    | 10 +++-
 src/server/status/lib/index.js                |  1 +
 src/server/status/lib/source_kibana.js        | 49 +++++++++++++++++++
 .../status/routes/api/register_stats.js       |  5 +-
 .../server/kibana_monitoring/bulk_uploader.js |  1 +
 8 files changed, 142 insertions(+), 2 deletions(-)
 create mode 100644 src/server/status/collectors/get_ops_stats_collector.js
 create mode 100644 src/server/status/collectors/index.js
 create mode 100644 src/server/status/constants.js
 create mode 100644 src/server/status/lib/source_kibana.js

diff --git a/src/server/status/collectors/get_ops_stats_collector.js b/src/server/status/collectors/get_ops_stats_collector.js
new file mode 100644
index 0000000000000..985138872bc1f
--- /dev/null
+++ b/src/server/status/collectors/get_ops_stats_collector.js
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import { KIBANA_STATS_TYPE } from '../constants';
+import { sourceKibana } from '../lib';
+
+/*
+ * Initialize a collector for Kibana Ops Stats
+ */
+export function getOpsStatsCollector(server, kbnServer) {
+  const { collectorSet } = server.usage;
+  return collectorSet.makeStatsCollector({
+    type: KIBANA_STATS_TYPE,
+    fetch: () => {
+      return {
+        kibana: sourceKibana(server, kbnServer),
+        ...kbnServer.metrics
+      };
+    }
+  });
+}
diff --git a/src/server/status/collectors/index.js b/src/server/status/collectors/index.js
new file mode 100644
index 0000000000000..4310dff7359ef
--- /dev/null
+++ b/src/server/status/collectors/index.js
@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { getOpsStatsCollector } from './get_ops_stats_collector';
diff --git a/src/server/status/constants.js b/src/server/status/constants.js
new file mode 100644
index 0000000000000..edd06b171a72c
--- /dev/null
+++ b/src/server/status/constants.js
@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const KIBANA_STATS_TYPE = 'kibana_stats';
diff --git a/src/server/status/index.js b/src/server/status/index.js
index 685bd3a4c095b..d4ab5f81ab7ac 100644
--- a/src/server/status/index.js
+++ b/src/server/status/index.js
@@ -20,17 +20,25 @@
 import ServerStatus from './server_status';
 import { Metrics } from './lib/metrics';
 import { registerStatusPage, registerStatusApi, registerStatsApi } from './routes';
+import { getOpsStatsCollector } from './collectors';
 
 export function statusMixin(kbnServer, server, config) {
   kbnServer.status = new ServerStatus(kbnServer.server);
 
+  const statsCollector = getOpsStatsCollector(server, kbnServer);
+
+  const { collectorSet } = server.usage;
+  collectorSet.register(statsCollector);
+
   const { ['even-better']: evenBetter } = server.plugins;
 
   if (evenBetter) {
     const metrics = new Metrics(config, server);
 
     evenBetter.monitor.on('ops', event => {
-      metrics.capture(event).then(data => { kbnServer.metrics = data; }); // for status API (to deprecate in next major)
+      metrics.capture(event).then(data => {
+        kbnServer.metrics = data;
+      });
     });
   }
 
diff --git a/src/server/status/lib/index.js b/src/server/status/lib/index.js
index abab4556677b5..195c2c8ab86e2 100644
--- a/src/server/status/lib/index.js
+++ b/src/server/status/lib/index.js
@@ -18,3 +18,4 @@
  */
 
 export { setApiFieldNames } from './set_api_field_names';
+export { sourceKibana } from './source_kibana';
diff --git a/src/server/status/lib/source_kibana.js b/src/server/status/lib/source_kibana.js
new file mode 100644
index 0000000000000..b4d12dde46419
--- /dev/null
+++ b/src/server/status/lib/source_kibana.js
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { get } from 'lodash';
+
+const snapshotRegex = /-snapshot/i;
+
+/**
+ * This provides a common structure to apply to all Kibana monitoring documents so that they can be commonly
+ * searched, field-collapsed, and aggregated against.
+ *
+ * 'sourceKibana' is akin to the `source_node` details in Elasticsearch nodes.
+ *
+ * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core
+ * @param {Object} config Server config
+ * @param {String} host Kibana host
+ * @return {Object} The object containing a "kibana" field and source instance details.
+ */
+export function sourceKibana(server, kbnServer) {
+  const config = server.config();
+  const status = kbnServer.status.toJSON();
+
+  return {
+    uuid: config.get('server.uuid'),
+    name: config.get('server.name'),
+    index: config.get('kibana.index'),
+    host: config.get('server.host'),
+    transport_address: `${config.get('server.host')}:${config.get('server.port')}`,
+    version: kbnServer.version.replace(snapshotRegex, ''),
+    snapshot: snapshotRegex.test(kbnServer.version),
+    status: get(status, 'overall.state')
+  };
+}
diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js
index 4a17245a210fb..a9afe310626ee 100644
--- a/src/server/status/routes/api/register_stats.js
+++ b/src/server/status/routes/api/register_stats.js
@@ -19,6 +19,7 @@
 
 import Joi from 'joi';
 import { wrapAuthConfig } from '../../wrap_auth_config';
+import { KIBANA_STATS_TYPE } from '../../constants';
 import { setApiFieldNames } from '../../lib';
 
 async function getExtended(req, server, collectorSet) {
@@ -36,6 +37,7 @@ async function getExtended(req, server, collectorSet) {
   let usage;
   try {
     const usageRaw = await collectorSet.bulkFetchUsage(callCluster);
+    // TODO put raw through composable functions
     usage = collectorSet.summarizeStats(usageRaw);
   } catch (err) {
     usage = undefined;
@@ -80,8 +82,9 @@ export function registerStatsApi(kbnServer, server, config) {
           ({ clusterUuid, usage } = await getExtended(req, server, collectorSet));
         }
 
-        const kibanaCollector = collectorSet.getCollectorByType('kibana_stats');
+        const kibanaCollector = collectorSet.getCollectorByType(KIBANA_STATS_TYPE);
         const stats = setApiFieldNames({
+          // TODO put raw through composable functions
           ...kibanaCollector.fetch(),
           cluster_uuid: clusterUuid,
           usage,
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
index bfd77b659b1fd..be71675dc506f 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
@@ -95,6 +95,7 @@ export class BulkUploader {
    */
   async _fetchAndUpload(collectorSet) {
     const data = await collectorSet.bulkFetch(this._callClusterWithInternalUser);
+    // TODO put the data through composable functions provided by collectorSet, with some overrides
     const payload = data
       .filter(d => Boolean(d) && !isEmpty(d.result))
       .map(({ result, type }) => [{ index: { _type: type } }, result]);

From b7f0766277a3ff9dcf841a15f36476508c6d18d2 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 11:49:47 -0700
Subject: [PATCH 07/42] isolate data transforms for api data and upload data

---
 src/server/status/lib/index.js                |  1 -
 src/server/status/lib/set_api_field_names.js  | 41 ---------
 .../status/routes/api/register_stats.js       | 65 ++++++-------
 src/server/usage/classes/collector_set.js     | 38 ++++++--
 .../server/kibana_monitoring/bulk_uploader.js | 91 +++++++++++++++----
 .../server/kibana_monitoring/init.js          |  6 +-
 .../lib/get_collector_types_combiner.js       | 78 ----------------
 .../server/kibana_monitoring/lib/index.js     |  1 -
 8 files changed, 132 insertions(+), 189 deletions(-)
 delete mode 100644 src/server/status/lib/set_api_field_names.js
 delete mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js

diff --git a/src/server/status/lib/index.js b/src/server/status/lib/index.js
index 195c2c8ab86e2..27f7f730d6cce 100644
--- a/src/server/status/lib/index.js
+++ b/src/server/status/lib/index.js
@@ -17,5 +17,4 @@
  * under the License.
  */
 
-export { setApiFieldNames } from './set_api_field_names';
 export { sourceKibana } from './source_kibana';
diff --git a/src/server/status/lib/set_api_field_names.js b/src/server/status/lib/set_api_field_names.js
deleted file mode 100644
index 590eec5c40a9a..0000000000000
--- a/src/server/status/lib/set_api_field_names.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-function getValueOrRecurse(value) {
-  if (value == null || typeof value !== 'object') {
-    return value;
-  } else {
-    return setApiFieldNames(value); // recurse
-  }
-}
-
-export function setApiFieldNames(apiData) {
-  return Object.keys(apiData).reduce((accum, currName) => {
-    const value = apiData[currName];
-
-    let newName = currName;
-    newName = newName.replace('_in_bytes', '_bytes');
-    newName = newName.replace('_in_millis', '_ms');
-
-    return {
-      ...accum,
-      [newName]: getValueOrRecurse(value),
-    };
-  }, {});
-}
diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js
index a9afe310626ee..30639e322b5b8 100644
--- a/src/server/status/routes/api/register_stats.js
+++ b/src/server/status/routes/api/register_stats.js
@@ -20,31 +20,7 @@
 import Joi from 'joi';
 import { wrapAuthConfig } from '../../wrap_auth_config';
 import { KIBANA_STATS_TYPE } from '../../constants';
-import { setApiFieldNames } from '../../lib';
-
-async function getExtended(req, server, collectorSet) {
-  const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('admin'); // admin cluster, get info on internal system
-  const callCluster = (...args) => callWithRequest(req, ...args);
-
-  let clusterUuid;
-  try {
-    const { cluster_uuid: uuid } = await callCluster('info', { filterPath: 'cluster_uuid', });
-    clusterUuid = uuid;
-  } catch (err) {
-    clusterUuid = undefined; // fallback from anonymous access or auth failure, redundant for explicitness
-  }
-
-  let usage;
-  try {
-    const usageRaw = await collectorSet.bulkFetchUsage(callCluster);
-    // TODO put raw through composable functions
-    usage = collectorSet.summarizeStats(usageRaw);
-  } catch (err) {
-    usage = undefined;
-  }
-
-  return { clusterUuid, usage };
-}
+import { CollectorSet } from '../../../usage/classes';
 
 /*
  * API for Kibana meta info and accumulated operations stats
@@ -57,9 +33,18 @@ async function getExtended(req, server, collectorSet) {
  */
 export function registerStatsApi(kbnServer, server, config) {
   const wrapAuth = wrapAuthConfig(config.get('status.allowAnonymous'));
-
   const { collectorSet } = server.usage;
 
+  const getClusterUuid = async callCluster => {
+    const { cluster_uuid: uuid } = await callCluster('info', { filterPath: 'cluster_uuid', });
+    return uuid;
+  };
+
+  const getUsage = async callCluster => {
+    const usage = await collectorSet.bulkFetchUsage(callCluster);
+    return CollectorSet.toApiStats(usage);
+  };
+
   server.route(
     wrapAuth({
       method: 'GET',
@@ -73,23 +58,27 @@ export function registerStatsApi(kbnServer, server, config) {
         tags: ['api'],
       },
       async handler(req, reply) {
-        const { extended } = req.query;
-        const isExtended = extended !== undefined && extended !== 'false';
+        const isExtended = req.query.extended !== undefined && req.query.extended !== 'false';
 
-        let clusterUuid;
-        let usage;
+        let extended;
         if (isExtended) {
-          ({ clusterUuid, usage } = await getExtended(req, server, collectorSet));
+          const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('admin');
+          const callCluster = (...args) => callWithRequest(req, ...args);
+          const [ usage, clusterUuid ] = await Promise.all([
+            getUsage(callCluster),
+            getClusterUuid(callCluster),
+          ]);
+          extended = { usage, clusterUuid };
         }
 
-        const kibanaCollector = collectorSet.getCollectorByType(KIBANA_STATS_TYPE);
-        const stats = setApiFieldNames({
-          // TODO put raw through composable functions
-          ...kibanaCollector.fetch(),
-          cluster_uuid: clusterUuid,
-          usage,
+        const kibanaStatsCollector = collectorSet.getCollectorByType(KIBANA_STATS_TYPE); // kibana stats get singled out
+        let kibanaStats = await kibanaStatsCollector.fetch();
+        kibanaStats = CollectorSet.setApiFieldNames(kibanaStats);
+
+        reply({
+          ...kibanaStats,
+          ...extended,
         });
-        reply(stats);
       },
     })
   );
diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js
index 0790c5b56f173..02ba87a85ed3e 100644
--- a/src/server/usage/classes/collector_set.js
+++ b/src/server/usage/classes/collector_set.js
@@ -94,19 +94,39 @@ export class CollectorSet {
   async bulkFetchUsage(callCluster) {
     const usageCollectors = this._collectors.filter(c => c instanceof UsageCollector);
     return this.bulkFetch(callCluster, usageCollectors);
-
   }
 
-  /*
-   * Summarize the data returned by bulk fetching into a simpler format
-   */
-  summarizeStats(statsData) {
-    return statsData.reduce((accumulatedStats, currentStat) => {
-      // `_stats` Suffix removal
-      const statType = currentStat.type.replace('_stats', '');
+  // convert the array of stats into key/object
+  static toApiStats(statsData) {
+    const summary = statsData.reduce((accumulatedStats, { type, result }) => {
       return {
         ...accumulatedStats,
-        [statType]: currentStat.result,
+        [type]: result,
+      };
+    }, {});
+    return CollectorSet.setApiFieldNames(summary);
+  }
+
+  // rename fields to use api conventions
+  static setApiFieldNames(apiData) {
+    const getValueOrRecurse = value => {
+      if (value == null || typeof value !== 'object') {
+        return value;
+      } else {
+        return CollectorSet.setApiFieldNames(value); // recurse
+      }
+    };
+
+    return Object.keys(apiData).reduce((accum, currName) => {
+      const value = apiData[currName];
+
+      let newName = currName;
+      newName = newName.replace('_in_bytes', '_bytes');
+      newName = newName.replace('_in_millis', '_ms');
+
+      return {
+        ...accum,
+        [newName]: getValueOrRecurse(value),
       };
     }, {});
   }
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
index be71675dc506f..cd9626289c93e 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
@@ -4,9 +4,16 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { isEmpty, flatten } from 'lodash';
+import { get, set, isEmpty, flatten } from 'lodash';
 import { callClusterFactory } from '../../../xpack_main';
-import { LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG } from '../../common/constants';
+import {
+  LOGGING_TAG,
+  KIBANA_MONITORING_LOGGING_TAG,
+  KIBANA_STATS_TYPE,
+  KIBANA_SETTINGS_TYPE,
+  KIBANA_USAGE_TYPE,
+} from '../../common/constants';
+import { KIBANA_REPORTING_TYPE } from '../../../reporting/common/constants';
 import {
   sendBulkPayload,
   monitoringBulk,
@@ -18,7 +25,7 @@ const LOGGING_TAGS = [LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG];
  * Handles internal Kibana stats collection and uploading data to Monitoring
  * bulk endpoint.
  *
- * NOTE: internal collection will be removed in 7.0
+ * TODO: remove this in 7.0
  *
  * Depends on
  *   - 'xpack.monitoring.kibana.collection.enabled' config
@@ -31,17 +38,13 @@ const LOGGING_TAGS = [LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG];
  * @param {Object} xpackInfo server.plugins.xpack_main.info object
  */
 export class BulkUploader {
-  constructor(server, { interval, combineTypes }) {
+  constructor(server, { interval }) {
     if (typeof interval !== 'number') {
       throw new Error('interval number of milliseconds is required');
     }
-    if (typeof combineTypes !== 'function') {
-      throw new Error('combineTypes function is required');
-    }
 
     this._timer =  null;
     this._interval = interval;
-    this._combineTypes = combineTypes;
     this._log = {
       debug: message => server.log(['debug', ...LOGGING_TAGS], message),
       info: message => server.log(['info', ...LOGGING_TAGS], message),
@@ -89,22 +92,22 @@ export class BulkUploader {
     this.stop('Connection issue detected');
   }
 
+  _onPayload(payload) {
+    return sendBulkPayload(this._client, this._interval, payload); // why is interval here?
+  }
+
   /*
    * @param {CollectorSet} collectorSet
    * @return {Promise} - resolves to undefined
    */
   async _fetchAndUpload(collectorSet) {
     const data = await collectorSet.bulkFetch(this._callClusterWithInternalUser);
-    // TODO put the data through composable functions provided by collectorSet, with some overrides
-    const payload = data
-      .filter(d => Boolean(d) && !isEmpty(d.result))
-      .map(({ result, type }) => [{ index: { _type: type } }, result]);
+    const uploadData = BulkUploader.legacyToUploadStats(data);
 
-    if (payload.length > 0) {
+    if (uploadData) {
       try {
-        const combinedData = this._combineTypes(payload); // use the collector types combiner to combine the data into a single bulk upload object
         this._log.debug(`Uploading bulk stats payload to the local cluster`);
-        this._onPayload(flatten(combinedData));
+        this._onPayload(uploadData);
       } catch (err) {
         this._log.warn(err.stack);
         this._log.warn(`Unable to bulk upload the stats payload to the local cluster`);
@@ -114,7 +117,61 @@ export class BulkUploader {
     }
   }
 
-  _onPayload(payload) {
-    return sendBulkPayload(this._client, this._interval, payload);
+  /*
+   * Bulk stats are transformed into a bulk upload format
+   * Non-legacy transformation is done in CollectorSet.toApiStats
+   */
+  static legacyToUploadStats(uploadData) {
+    const payload = uploadData
+      .filter(d => Boolean(d) && !isEmpty(d.result))
+      .map(({ result, type }) => [{ index: { _type: type } }, result]);
+    if (payload.length > 0) {
+      const combinedData = BulkUploader.legacyCombineTypes(payload); // arrange the usage data into the stats
+      return flatten(combinedData);
+    }
+  }
+
+  static legacyCombineTypes(payload) {
+    // default the item to [] to allow destructuring
+    const findItem = type => payload.find(item => get(item, '[0].index._type') === type) || [];
+
+    // kibana usage and stats
+    let statsResult;
+    const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE);
+    const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE);
+
+    if (statsHeader && statsPayload) {
+      const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE);
+      const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null;
+      const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null;
+      statsResult = [
+        statsHeader,
+        statsPayload
+      ];
+      if (kibanaUsage) {
+        set(statsResult, '[1].usage', kibanaUsage);
+      }
+      if (reportingUsage) {
+        set(statsResult, '[1].usage.xpack.reporting', reportingUsage);
+      }
+    }
+
+    // kibana settings
+    let settingsResult;
+    const [ settingsHeader, settingsPayload ] = findItem(KIBANA_SETTINGS_TYPE);
+    if (settingsHeader && settingsPayload) {
+      settingsResult = [
+        settingsHeader,
+        settingsPayload,
+      ];
+    }
+
+    // return new payload with the combined data
+    // adds usage data to stats data
+    // strips usage out as a top-level type
+    const result = [ statsResult, settingsResult ];
+
+    // remove result items that are undefined
+    return result.filter(Boolean);
   }
 }
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js
index 3e136c5226cbc..3e92d4d7c959d 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js
@@ -5,7 +5,6 @@
  */
 
 import { BulkUploader } from './bulk_uploader';
-import { getCollectorTypesCombiner } from './lib';
 
 /**
  * Initialize different types of Kibana Monitoring
@@ -16,12 +15,11 @@ import { getCollectorTypesCombiner } from './lib';
  * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core
  * @param {Object} server HapiJS server instance
  */
-export function initBulkUploader(kbnServer, server) {
+export function initBulkUploader(_kbnServer, server) {
 
   const config = server.config();
   const interval = config.get('xpack.monitoring.kibana.collection.interval');
   return new BulkUploader(server, {
-    interval,
-    combineTypes: getCollectorTypesCombiner(kbnServer, config)
+    interval
   });
 }
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js
deleted file mode 100644
index 4e029fff0edfc..0000000000000
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_collector_types_combiner.js
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { get, set } from 'lodash';
-import {
-  KIBANA_STATS_TYPE,
-  KIBANA_SETTINGS_TYPE,
-  KIBANA_USAGE_TYPE,
-} from '../../../common/constants';
-import { KIBANA_REPORTING_TYPE } from '../../../../reporting/common/constants';
-
-/*
- * Combine stats collected from different sources into a single bulk payload.
- *
- * The ES Bulk Data Format is an array with 2 objects:
- * - The first object is the header, it has a field for the action (index), and
- *   metadata of the document (_index, _type, _id).
- * - The second object is the actual document to index.
- *
- * NOTE: https://github.com/elastic/kibana/issues/12504 asks that plugins have
- * a way to register their own stats. It's not hard to move the stats collector
- * methods under the ownership of the plugins that want it, but this module's
- * behavior doesn't fit well with plugins registering their own stats. See the
- * abstraction leak comments in the code.
- *
- * This module should go away when stats are collected by a Kibana metricbeat moduleset.
- *  - Individual plugin operational stats can be added to the `/stats?extended` API response.
- *  - Individual plugin usage stats can go into a new API similar to the `_xpack/usage` API in ES.
- *  - Each plugin will have its own top-level property in the responses for these APIs.
- */
-export function getCollectorTypesCombiner() {
-  return payload => {
-    // default the item to [] to allow destructuring
-    const findItem = type => payload.find(item => get(item, '[0].index._type') === type) || [];
-
-    // kibana usage and stats
-    let statsResult;
-    const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE);
-    const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE);
-
-    if (statsHeader && statsPayload) {
-      const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE);
-      const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null;
-      const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null; // this is an abstraction leak
-      statsResult = [
-        statsHeader,
-        statsPayload
-      ];
-      if (kibanaUsage) {
-        set(statsResult, '[1].usage', kibanaUsage);
-      }
-      if (reportingUsage) {
-        set(statsResult, '[1].usage.xpack.reporting', reportingUsage); // this is an abstraction leak
-      }
-    }
-
-    // kibana settings
-    let settingsResult;
-    const [ settingsHeader, settingsPayload ] = findItem(KIBANA_SETTINGS_TYPE);
-    if (settingsHeader && settingsPayload) {
-      settingsResult = [
-        settingsHeader,
-        settingsPayload,
-      ];
-    }
-
-    // return new payload with the combined data
-    // adds usage data to stats data
-    // strips usage out as a top-level type
-    const result = [ statsResult, settingsResult ];
-
-    // remove result items that are undefined
-    return result.filter(Boolean);
-  };
-}
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js
index ea4fb99abf0ea..0b68a0aed0d2f 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js
@@ -4,7 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export { getCollectorTypesCombiner } from './get_collector_types_combiner';
 export { sendBulkPayload } from './send_bulk_payload';
 export { monitoringBulk } from './monitoring_bulk';
 export { sourceKibana } from './source_kibana';

From fb05f2026b471ed471077f3fb79b90fe66aa4f9b Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 12:35:49 -0700
Subject: [PATCH 08/42] no static methods

---
 src/server/status/routes/api/register_stats.js       | 12 ++++++++----
 src/server/usage/classes/collector_set.js            | 11 +++++------
 .../server/kibana_monitoring/bulk_uploader.js        |  8 ++++----
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js
index 30639e322b5b8..504c0ca0b7b9c 100644
--- a/src/server/status/routes/api/register_stats.js
+++ b/src/server/status/routes/api/register_stats.js
@@ -20,7 +20,6 @@
 import Joi from 'joi';
 import { wrapAuthConfig } from '../../wrap_auth_config';
 import { KIBANA_STATS_TYPE } from '../../constants';
-import { CollectorSet } from '../../../usage/classes';
 
 /*
  * API for Kibana meta info and accumulated operations stats
@@ -42,7 +41,8 @@ export function registerStatsApi(kbnServer, server, config) {
 
   const getUsage = async callCluster => {
     const usage = await collectorSet.bulkFetchUsage(callCluster);
-    return CollectorSet.toApiStats(usage);
+    const usageObject = collectorSet.toObject(usage);
+    return collectorSet.toApiFieldNames(usageObject);
   };
 
   server.route(
@@ -71,9 +71,13 @@ export function registerStatsApi(kbnServer, server, config) {
           extended = { usage, clusterUuid };
         }
 
-        const kibanaStatsCollector = collectorSet.getCollectorByType(KIBANA_STATS_TYPE); // kibana stats get singled out
+        /* kibana_stats gets singled out from the collector set as it is used
+         * for health-checking Kibana and fetch does not rely on fetching data
+         * from ES
+         */
+        const kibanaStatsCollector = collectorSet.getCollectorByType(KIBANA_STATS_TYPE);
         let kibanaStats = await kibanaStatsCollector.fetch();
-        kibanaStats = CollectorSet.setApiFieldNames(kibanaStats);
+        kibanaStats = collectorSet.toApiFieldNames(kibanaStats);
 
         reply({
           ...kibanaStats,
diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js
index 02ba87a85ed3e..640c6a27f5faa 100644
--- a/src/server/usage/classes/collector_set.js
+++ b/src/server/usage/classes/collector_set.js
@@ -96,24 +96,23 @@ export class CollectorSet {
     return this.bulkFetch(callCluster, usageCollectors);
   }
 
-  // convert the array of stats into key/object
-  static toApiStats(statsData) {
-    const summary = statsData.reduce((accumulatedStats, { type, result }) => {
+  // convert an array of fetched stats results into key/object
+  toObject(statsData) {
+    return statsData.reduce((accumulatedStats, { type, result }) => {
       return {
         ...accumulatedStats,
         [type]: result,
       };
     }, {});
-    return CollectorSet.setApiFieldNames(summary);
   }
 
   // rename fields to use api conventions
-  static setApiFieldNames(apiData) {
+  toApiFieldNames(apiData) {
     const getValueOrRecurse = value => {
       if (value == null || typeof value !== 'object') {
         return value;
       } else {
-        return CollectorSet.setApiFieldNames(value); // recurse
+        return this.toApiFieldNames(value); // recurse
       }
     };
 
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
index cd9626289c93e..e9de990426c71 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
@@ -102,7 +102,7 @@ export class BulkUploader {
    */
   async _fetchAndUpload(collectorSet) {
     const data = await collectorSet.bulkFetch(this._callClusterWithInternalUser);
-    const uploadData = BulkUploader.legacyToUploadStats(data);
+    const uploadData = this.toBulkUploadFormat(data);
 
     if (uploadData) {
       try {
@@ -121,17 +121,17 @@ export class BulkUploader {
    * Bulk stats are transformed into a bulk upload format
    * Non-legacy transformation is done in CollectorSet.toApiStats
    */
-  static legacyToUploadStats(uploadData) {
+  toBulkUploadFormat(uploadData) {
     const payload = uploadData
       .filter(d => Boolean(d) && !isEmpty(d.result))
       .map(({ result, type }) => [{ index: { _type: type } }, result]);
     if (payload.length > 0) {
-      const combinedData = BulkUploader.legacyCombineTypes(payload); // arrange the usage data into the stats
+      const combinedData = this.combineStatsLegacy(payload); // arrange the usage data into the stats
       return flatten(combinedData);
     }
   }
 
-  static legacyCombineTypes(payload) {
+  combineStatsLegacy(payload) {
     // default the item to [] to allow destructuring
     const findItem = type => payload.find(item => get(item, '[0].index._type') === type) || [];
 

From 4fe1c9d98a5be5bf36795bbbae6e844aafd7f9d0 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 15:06:39 -0700
Subject: [PATCH 09/42] remove external in bytes

---
 src/server/status/lib/metrics.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/server/status/lib/metrics.js b/src/server/status/lib/metrics.js
index bc404ae628b5f..ffd98e5c3f2e7 100644
--- a/src/server/status/lib/metrics.js
+++ b/src/server/status/lib/metrics.js
@@ -75,7 +75,6 @@ export class Metrics {
           heap_max_in_bytes: get(hapiEvent, 'psmem.heapTotal'),
           heap_used_in_bytes: get(hapiEvent, 'psmem.heapUsed'),
           resident_set_size_in_bytes: get(hapiEvent, 'psmem.rss'),
-          external_in_bytes: get(hapiEvent, 'psmem.external')
         },
         pid: process.pid,
         uptime_ms: process.uptime() * 1000

From c1e8226d8c10a970afc02ac5eb9e19a504966b6e Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 15:08:04 -0700
Subject: [PATCH 10/42] remove the _stats prefix for kibana and reporting

---
 src/server/status/constants.js               | 2 +-
 x-pack/plugins/reporting/common/constants.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/server/status/constants.js b/src/server/status/constants.js
index edd06b171a72c..1bd84dd1e4b03 100644
--- a/src/server/status/constants.js
+++ b/src/server/status/constants.js
@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export const KIBANA_STATS_TYPE = 'kibana_stats';
+export const KIBANA_STATS_TYPE = 'kibana';
diff --git a/x-pack/plugins/reporting/common/constants.js b/x-pack/plugins/reporting/common/constants.js
index 6376c44042e25..09321bd8d3c3e 100644
--- a/x-pack/plugins/reporting/common/constants.js
+++ b/x-pack/plugins/reporting/common/constants.js
@@ -18,4 +18,4 @@ export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo';
  * The type name used within the Monitoring index to publish reporting stats.
  * @type {string}
  */
-export const KIBANA_REPORTING_TYPE = 'reporting_stats';
+export const KIBANA_REPORTING_TYPE = 'reporting';

From d162284b54ceb31f8f1ac910bb00ef6e9eebd96c Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 15:19:06 -0700
Subject: [PATCH 11/42] update jest test snapshot

---
 .../lib/__snapshots__/metrics.test.js.snap    | 112 ++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 src/server/status/lib/__snapshots__/metrics.test.js.snap

diff --git a/src/server/status/lib/__snapshots__/metrics.test.js.snap b/src/server/status/lib/__snapshots__/metrics.test.js.snap
new file mode 100644
index 0000000000000..ac22c0ba72d40
--- /dev/null
+++ b/src/server/status/lib/__snapshots__/metrics.test.js.snap
@@ -0,0 +1,112 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Metrics capture merges all metrics 1`] = `
+Object {
+  "a": Array [
+    Object {
+      "b": 2,
+      "c": 3,
+    },
+    Object {
+      "d": 4,
+      "e": 5,
+    },
+  ],
+  "collection_interval_in_millis": 5000,
+  "last_updated": "2017-04-14T18:35:41.534Z",
+  "process": Object {
+    "uptime_ms": 1980,
+  },
+}
+`;
+
+exports[`Metrics captureEvent parses event with missing fields / NaN for responseTimes.avg 1`] = `
+Object {
+  "concurrent_connections": undefined,
+  "event_loop_delay": undefined,
+  "os": Object {
+    "cpu": Object {
+      "load_average": Object {
+        "15m": undefined,
+        "1m": undefined,
+        "5m": undefined,
+      },
+    },
+    "mem": Object {
+      "free_in_bytes": 12,
+      "total_in_bytes": 24,
+    },
+    "uptime_ms": 12000000,
+  },
+  "process": Object {
+    "mem": Object {
+      "heap_max_in_bytes": undefined,
+      "heap_used_in_bytes": undefined,
+      "resident_set_size_in_bytes": undefined,
+    },
+    "pid": 8675309,
+    "uptime_ms": 5000000,
+  },
+  "requests": Object {
+    "disconnects": 0,
+    "status_codes": Object {
+      "200": 22,
+    },
+    "total": 22,
+  },
+  "response_times": Object {
+    "avg_in_millis": undefined,
+    "max_in_millis": 4,
+  },
+  "sockets": undefined,
+}
+`;
+
+exports[`Metrics captureEvent parses the hapi event 1`] = `
+Object {
+  "concurrent_connections": 0,
+  "event_loop_delay": 1.6091690063476562,
+  "os": Object {
+    "cpu": Object {
+      "load_average": Object {
+        "15m": 1.89794921875,
+        "1m": 2.20751953125,
+        "5m": 2.02294921875,
+      },
+    },
+    "mem": Object {
+      "free_in_bytes": 12,
+      "total_in_bytes": 24,
+    },
+    "uptime_ms": 12000000,
+  },
+  "process": Object {
+    "mem": Object {
+      "heap_max_in_bytes": 168194048,
+      "heap_used_in_bytes": 130553400,
+      "resident_set_size_in_bytes": 193716224,
+    },
+    "pid": 8675309,
+    "uptime_ms": 5000000,
+  },
+  "requests": Object {
+    "disconnects": 0,
+    "status_codes": Object {
+      "200": 22,
+    },
+    "total": 22,
+  },
+  "response_times": Object {
+    "avg_in_millis": 1.8636363636363635,
+    "max_in_millis": 4,
+  },
+  "sockets": Object {
+    "http": Object {
+      "total": 0,
+    },
+    "https": Object {
+      "total": 0,
+    },
+  },
+}
+`;

From 489be425d292843951c5ac01895086ebb99aa53e Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 15:35:07 -0700
Subject: [PATCH 12/42] fix collector_types_combiner test

---
 ...ulk_uploader.combine_stats_legacy.test.js} | 61 ++++---------------
 .../server/kibana_monitoring/bulk_uploader.js |  8 +--
 2 files changed, 17 insertions(+), 52 deletions(-)
 rename x-pack/plugins/monitoring/server/kibana_monitoring/{lib/__tests__/get_collector_types_combiner.js => bulk_uploader.combine_stats_legacy.test.js} (72%)

diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/__tests__/get_collector_types_combiner.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js
similarity index 72%
rename from x-pack/plugins/monitoring/server/kibana_monitoring/lib/__tests__/get_collector_types_combiner.js
rename to x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js
index 4bf5c2a37c9a6..e02d5d45f37fb 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/__tests__/get_collector_types_combiner.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js
@@ -4,8 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { getCollectorTypesCombiner } from '../get_collector_types_combiner';
-import expect from 'expect.js';
+import { KIBANA_REPORTING_TYPE } from '../../../reporting/common/constants';
+import { BulkUploader } from './bulk_uploader';
 
 const getInitial = () => {
   return [
@@ -47,7 +47,7 @@ const getInitial = () => {
       }
     ],
     [
-      { 'index': { '_type': 'reporting_stats' } },
+      { 'index': { '_type': KIBANA_REPORTING_TYPE } },
       {
         'available': true,
         'enabled': false,
@@ -69,11 +69,13 @@ const getInitial = () => {
   ];
 };
 
+// TODO use jest snapshotting
 const getResult = () => {
   return [
     [
       { 'index': { '_type': 'kibana_stats' } },
       {
+        'host': 'tsullivan.local',
         'concurrent_connections': 0,
         'os': {
           'load': { '1m': 2.28857421875, '5m': 2.45068359375, '15m': 2.29248046875 },
@@ -95,16 +97,6 @@ const getResult = () => {
         },
         'response_times': { 'average': 47, 'max': 47 },
         'timestamp': '2017-07-26T00:14:20.771Z',
-        'kibana': {
-          'uuid': '5b2de169-2785-441b-ae8c-186a1936b17d',
-          'name': 'tsullivan.local',
-          'index': '.kibana',
-          'host': 'tsullivan.local',
-          'transport_address': 'tsullivan.local:5601',
-          'version': '6.0.0-beta1',
-          'snapshot': false,
-          'status': 'green'
-        },
         'usage': {
           'dashboard': { 'total': 0 },
           'visualization': { 'total': 0 },
@@ -133,42 +125,18 @@ const getResult = () => {
       { 'index': { '_type': 'kibana_settings' } },
       {
         'xpack': { 'defaultAdminEmail': 'tim@elastic.co' },
-        'kibana': {
-          'uuid': '5b2de169-2785-441b-ae8c-186a1936b17d',
-          'name': 'tsullivan.local',
-          'index': '.kibana',
-          'host': 'tsullivan.local',
-          'transport_address': 'tsullivan.local:5601',
-          'version': '6.0.0-beta1',
-          'snapshot': false,
-          'status': 'green'
-        }
       }
     ]
   ];
 };
 
-const kbnServerMock = {};
-const configMock = {};
-const sourceKibanaMock = () => ({
-  uuid: '5b2de169-2785-441b-ae8c-186a1936b17d',
-  name: 'tsullivan.local',
-  index: '.kibana',
-  host: 'tsullivan.local',
-  transport_address: 'tsullivan.local:5601',
-  version: '6.0.0-beta1',
-  snapshot: false,
-  status: 'green'
-});
-
 describe('Collector Types Combiner', () => {
   describe('with all the data types present', () => {
     it('provides settings, and combined stats/usage data', () => {
       // default gives all the data types
       const initial = getInitial();
-      const combiner = getCollectorTypesCombiner(kbnServerMock, configMock, sourceKibanaMock);
-      const result = combiner(initial);
-      expect(result).to.eql(getResult());
+      const result = BulkUploader.combineStatsLegacy(initial);
+      expect(result).toEqual(getResult());
     });
   });
   describe('with settings data missing', () => {
@@ -176,11 +144,10 @@ describe('Collector Types Combiner', () => {
       // default gives all the data types
       const initial = getInitial();
       const trimmedInitial = [ initial[0], initial[1], initial[2] ]; // just stats, usage and reporting, no settings
-      const combiner = getCollectorTypesCombiner(kbnServerMock, configMock, sourceKibanaMock);
-      const result = combiner(trimmedInitial);
+      const result = BulkUploader.combineStatsLegacy(trimmedInitial);
       const expectedResult = getResult();
       const trimmedExpectedResult = [ expectedResult[0] ]; // single combined item
-      expect(result).to.eql(trimmedExpectedResult);
+      expect(result).toEqual(trimmedExpectedResult);
     });
   });
   describe('with usage data missing', () => {
@@ -188,12 +155,11 @@ describe('Collector Types Combiner', () => {
       // default gives all the data types
       const initial = getInitial();
       const trimmedInitial = [ initial[0], initial[3] ]; // just stats and settings, no usage or reporting
-      const combiner = getCollectorTypesCombiner(kbnServerMock, configMock, sourceKibanaMock);
-      const result = combiner(trimmedInitial);
+      const result = BulkUploader.combineStatsLegacy(trimmedInitial);
       const expectedResult = getResult();
       delete expectedResult[0][1].usage; // usage stats should not be present in the result
       const trimmedExpectedResult = [ expectedResult[0], expectedResult[1] ];
-      expect(result).to.eql(trimmedExpectedResult);
+      expect(result).toEqual(trimmedExpectedResult);
     });
   });
   describe('with stats data missing', () => {
@@ -201,11 +167,10 @@ describe('Collector Types Combiner', () => {
       // default gives all the data types
       const initial = getInitial();
       const trimmedInitial = [ initial[3] ]; // just settings
-      const combiner = getCollectorTypesCombiner(kbnServerMock, configMock, sourceKibanaMock);
-      const result = combiner(trimmedInitial);
+      const result = BulkUploader.combineStatsLegacy(trimmedInitial);
       const expectedResult = getResult();
       const trimmedExpectedResult = [ expectedResult[1] ]; // just settings
-      expect(result).to.eql(trimmedExpectedResult);
+      expect(result).toEqual(trimmedExpectedResult);
     });
   });
 });
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
index e9de990426c71..2c7e4520f80a2 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
@@ -102,7 +102,7 @@ export class BulkUploader {
    */
   async _fetchAndUpload(collectorSet) {
     const data = await collectorSet.bulkFetch(this._callClusterWithInternalUser);
-    const uploadData = this.toBulkUploadFormat(data);
+    const uploadData = BulkUploader.toBulkUploadFormat(data);
 
     if (uploadData) {
       try {
@@ -121,17 +121,17 @@ export class BulkUploader {
    * Bulk stats are transformed into a bulk upload format
    * Non-legacy transformation is done in CollectorSet.toApiStats
    */
-  toBulkUploadFormat(uploadData) {
+  static toBulkUploadFormat(uploadData) {
     const payload = uploadData
       .filter(d => Boolean(d) && !isEmpty(d.result))
       .map(({ result, type }) => [{ index: { _type: type } }, result]);
     if (payload.length > 0) {
-      const combinedData = this.combineStatsLegacy(payload); // arrange the usage data into the stats
+      const combinedData = BulkUploader.combineStatsLegacy(payload); // arrange the usage data into the stats
       return flatten(combinedData);
     }
   }
 
-  combineStatsLegacy(payload) {
+  static combineStatsLegacy(payload) {
     // default the item to [] to allow destructuring
     const findItem = type => payload.find(item => get(item, '[0].index._type') === type) || [];
 

From 817314c57bf3b8ed26af9231f80e427a93a3e883 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 15:54:41 -0700
Subject: [PATCH 13/42] fix usage api

---
 x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js b/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js
index a1e6b4fd2ab9a..ea01d119efe7f 100644
--- a/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js
+++ b/x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js
@@ -18,7 +18,8 @@ const getClusterUuid = async callCluster => {
 const getUsage = async (callCluster, server) => {
   const { collectorSet } = server.usage;
   const usage = await collectorSet.bulkFetchUsage(callCluster);
-  return collectorSet.summarizeStats(usage);
+  const usageObject = collectorSet.toObject(usage);
+  return collectorSet.toApiFieldNames(usageObject);
 };
 
 export function xpackUsageRoute(server) {

From dfb901b2e093c38dbd6697f04b10ced32c2e2ae9 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 16:04:40 -0700
Subject: [PATCH 14/42] add test suite todo comment

---
 test/api_integration/apis/stats/stats.js | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/test/api_integration/apis/stats/stats.js b/test/api_integration/apis/stats/stats.js
index 2c9c056884322..dfcf5198b291a 100644
--- a/test/api_integration/apis/stats/stats.js
+++ b/test/api_integration/apis/stats/stats.js
@@ -59,7 +59,7 @@ export default function ({ getService }) {
 
   describe('kibana stats api', () => {
     describe('basic', () => {
-      it('should return the stats and metric fields without cluster_uuid when extended param is not present', () => {
+      it('should return the stats without cluster_uuid with no query string params', () => {
         return supertest
           .get('/api/stats')
           .expect('Content-Type', /json/)
@@ -69,7 +69,7 @@ export default function ({ getService }) {
             assertStatsAndMetrics(body);
           });
       });
-      it('should return the stats and metric fields without cluster_uuid when extended param is given as false', () => {
+      it(`should return the stats without cluster_uuid with 'extended' query string param = false`, () => {
         return supertest
           .get('/api/stats?extended=false')
           .expect('Content-Type', /json/)
@@ -81,8 +81,9 @@ export default function ({ getService }) {
       });
     });
 
+    // TODO load an es archive and verify the counts in saved object usage info
     describe('extended', () => {
-      it('should return the stats and metric fields with cluster_uuid when extended param is present', () => {
+      it(`should return the stats, cluster_uuid, and usage with 'extended' query string param present`, () => {
         return supertest
           .get('/api/stats?extended')
           .expect('Content-Type', /json/)
@@ -94,7 +95,7 @@ export default function ({ getService }) {
           });
       });
 
-      it('should return the stats and metric fields with cluster_uuid when extended param is given as true', () => {
+      it(`should return the stats, cluster_uuid, and usage with 'extended' query string param = true`, () => {
         return supertest
           .get('/api/stats?extended=true')
           .expect('Content-Type', /json/)

From 48d4987b7b96f6ae35e4cf803a2a1bd7820d463b Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 16:45:06 -0700
Subject: [PATCH 15/42] reduce some loc change

---
 src/server/status/index.js                    |   4 +-
 .../lib/__snapshots__/metrics.test.js.snap    | 112 ------------------
 src/server/status/lib/metrics.js              |   1 +
 src/server/status/lib/metrics.test.js         |  67 ++++++++++-
 4 files changed, 65 insertions(+), 119 deletions(-)
 delete mode 100644 src/server/status/lib/__snapshots__/metrics.test.js.snap

diff --git a/src/server/status/index.js b/src/server/status/index.js
index d4ab5f81ab7ac..cc332b8ffebc7 100644
--- a/src/server/status/index.js
+++ b/src/server/status/index.js
@@ -36,9 +36,7 @@ export function statusMixin(kbnServer, server, config) {
     const metrics = new Metrics(config, server);
 
     evenBetter.monitor.on('ops', event => {
-      metrics.capture(event).then(data => {
-        kbnServer.metrics = data;
-      });
+      metrics.capture(event).then(data => { kbnServer.metrics = data; });
     });
   }
 
diff --git a/src/server/status/lib/__snapshots__/metrics.test.js.snap b/src/server/status/lib/__snapshots__/metrics.test.js.snap
deleted file mode 100644
index ac22c0ba72d40..0000000000000
--- a/src/server/status/lib/__snapshots__/metrics.test.js.snap
+++ /dev/null
@@ -1,112 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`Metrics capture merges all metrics 1`] = `
-Object {
-  "a": Array [
-    Object {
-      "b": 2,
-      "c": 3,
-    },
-    Object {
-      "d": 4,
-      "e": 5,
-    },
-  ],
-  "collection_interval_in_millis": 5000,
-  "last_updated": "2017-04-14T18:35:41.534Z",
-  "process": Object {
-    "uptime_ms": 1980,
-  },
-}
-`;
-
-exports[`Metrics captureEvent parses event with missing fields / NaN for responseTimes.avg 1`] = `
-Object {
-  "concurrent_connections": undefined,
-  "event_loop_delay": undefined,
-  "os": Object {
-    "cpu": Object {
-      "load_average": Object {
-        "15m": undefined,
-        "1m": undefined,
-        "5m": undefined,
-      },
-    },
-    "mem": Object {
-      "free_in_bytes": 12,
-      "total_in_bytes": 24,
-    },
-    "uptime_ms": 12000000,
-  },
-  "process": Object {
-    "mem": Object {
-      "heap_max_in_bytes": undefined,
-      "heap_used_in_bytes": undefined,
-      "resident_set_size_in_bytes": undefined,
-    },
-    "pid": 8675309,
-    "uptime_ms": 5000000,
-  },
-  "requests": Object {
-    "disconnects": 0,
-    "status_codes": Object {
-      "200": 22,
-    },
-    "total": 22,
-  },
-  "response_times": Object {
-    "avg_in_millis": undefined,
-    "max_in_millis": 4,
-  },
-  "sockets": undefined,
-}
-`;
-
-exports[`Metrics captureEvent parses the hapi event 1`] = `
-Object {
-  "concurrent_connections": 0,
-  "event_loop_delay": 1.6091690063476562,
-  "os": Object {
-    "cpu": Object {
-      "load_average": Object {
-        "15m": 1.89794921875,
-        "1m": 2.20751953125,
-        "5m": 2.02294921875,
-      },
-    },
-    "mem": Object {
-      "free_in_bytes": 12,
-      "total_in_bytes": 24,
-    },
-    "uptime_ms": 12000000,
-  },
-  "process": Object {
-    "mem": Object {
-      "heap_max_in_bytes": 168194048,
-      "heap_used_in_bytes": 130553400,
-      "resident_set_size_in_bytes": 193716224,
-    },
-    "pid": 8675309,
-    "uptime_ms": 5000000,
-  },
-  "requests": Object {
-    "disconnects": 0,
-    "status_codes": Object {
-      "200": 22,
-    },
-    "total": 22,
-  },
-  "response_times": Object {
-    "avg_in_millis": 1.8636363636363635,
-    "max_in_millis": 4,
-  },
-  "sockets": Object {
-    "http": Object {
-      "total": 0,
-    },
-    "https": Object {
-      "total": 0,
-    },
-  },
-}
-`;
diff --git a/src/server/status/lib/metrics.js b/src/server/status/lib/metrics.js
index ffd98e5c3f2e7..ec3c115de1705 100644
--- a/src/server/status/lib/metrics.js
+++ b/src/server/status/lib/metrics.js
@@ -56,6 +56,7 @@ export class Metrics {
 
     const metrics = {
       last_updated: timestamp,
+      uptime_in_millis: event.process.uptime_ms, // TODO: deprecate this field, data should only have process.uptime_ms
       collection_interval_in_millis: this.config.get('ops.interval')
     };
 
diff --git a/src/server/status/lib/metrics.test.js b/src/server/status/lib/metrics.test.js
index ed82a15db3a1b..14c2d4932909a 100644
--- a/src/server/status/lib/metrics.test.js
+++ b/src/server/status/lib/metrics.test.js
@@ -72,7 +72,12 @@ describe('Metrics', function () {
       sinon.stub(Date.prototype, 'toISOString').returns('2017-04-14T18:35:41.534Z');
 
       const capturedMetrics = await metrics.capture();
-      expect(capturedMetrics).toMatchSnapshot();
+      expect(capturedMetrics).toMatchObject({
+        last_updated: '2017-04-14T18:35:41.534Z',
+        collection_interval_in_millis: 5000,
+        uptime_in_millis: 1980,
+        a: [ { b: 2, c: 3 }, { d: 4, e: 5 } ], process: { uptime_ms: 1980 }
+      });
     });
   });
 
@@ -105,7 +110,52 @@ describe('Metrics', function () {
         'host': 'blahblah.local'
       };
 
-      expect(metrics.captureEvent(hapiEvent)).toMatchSnapshot();
+      expect(metrics.captureEvent(hapiEvent)).toMatchObject({
+        'concurrent_connections': 0,
+        'event_loop_delay': 1.6091690063476562,
+        'os': {
+          'cpu': {
+            'load_average': {
+              '15m': 1.89794921875,
+              '1m': 2.20751953125,
+              '5m': 2.02294921875
+            }
+          },
+          'mem': {
+            'free_in_bytes': 12,
+            'total_in_bytes': 24,
+          },
+        },
+        'process': {
+          'mem': {
+            'heap_max_in_bytes': 168194048,
+            'heap_used_in_bytes': 130553400,
+            'resident_set_size_in_bytes': 193716224,
+          },
+          'pid': 8675309,
+          'uptime_ms': 5000000
+        },
+        'requests': {
+          'disconnects': 0,
+          'status_codes': {
+            '200': 22
+          },
+          'total': 22
+        },
+        'response_times': {
+          'avg_in_millis': 1.8636363636363635,
+          'max_in_millis': 4
+        },
+        'sockets': {
+          'http': {
+            'total': 0
+          },
+          'https': {
+            'total': 0
+          }
+        }
+      });
+
     });
 
     it('parses event with missing fields / NaN for responseTimes.avg', () => {
@@ -117,7 +167,16 @@ describe('Metrics', function () {
         host: 'blahblah.local',
       };
 
-      expect(metrics.captureEvent(hapiEvent)).toMatchSnapshot();
+      expect(metrics.captureEvent(hapiEvent)).toMatchObject({
+        process: { mem: {}, pid: 8675309, uptime_ms: 5000000 },
+        os: {
+          cpu: { load_average: {} },
+          mem: { free_in_bytes: 12, total_in_bytes: 24 },
+        },
+        response_times: { max_in_millis: 4 },
+        requests: { total: 22, disconnects: 0, status_codes: { '200': 22 } },
+      });
+
     });
   });
 
@@ -140,7 +199,7 @@ describe('Metrics', function () {
 
       const capturedMetrics = await metrics.captureCGroups();
 
-      expect(capturedMetrics).toEqual({
+      expect(capturedMetrics).toMatchObject({
         os: {
           cgroup: {
             cpuacct: {

From 9be60c90b3096debc04aefe7084262d8f5327295 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 16:52:03 -0700
Subject: [PATCH 16/42] roll back mysterious change

---
 src/server/status/routes/api/register_status.js | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/server/status/routes/api/register_status.js b/src/server/status/routes/api/register_status.js
index 64f23170a44fb..98c454de4f58d 100644
--- a/src/server/status/routes/api/register_status.js
+++ b/src/server/status/routes/api/register_status.js
@@ -41,10 +41,7 @@ export function registerStatusApi(kbnServer, server, config) {
           build_snapshot: matchSnapshot.test(config.get('pkg.version'))
         },
         status: kbnServer.status.toJSON(),
-        metrics: {
-          ...kbnServer.metrics,
-          uptime_in_millis: process.uptime() * 1000
-        }
+        metrics: kbnServer.metrics
       };
 
       return reply(status);

From 244627ff6c9f791e76ab0d55f344f43ff1245d93 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 16:52:22 -0700
Subject: [PATCH 17/42] reduce some more loc change

---
 test/api_integration/apis/stats/stats.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/api_integration/apis/stats/stats.js b/test/api_integration/apis/stats/stats.js
index dfcf5198b291a..0d0f8169d119f 100644
--- a/test/api_integration/apis/stats/stats.js
+++ b/test/api_integration/apis/stats/stats.js
@@ -21,25 +21,25 @@ import expect from 'expect.js';
 
 const assertStatsAndMetrics = body => {
   expect(body.kibana.name).to.be.a('string');
-  expect(body.kibana.host).to.be.a('string');
   expect(body.kibana.uuid).to.be.a('string');
+  expect(body.kibana.host).to.be.a('string');
   expect(body.kibana.transport_address).to.be.a('string');
   expect(body.kibana.version).to.be.a('string');
   expect(body.kibana.snapshot).to.be.a('boolean');
   expect(body.kibana.status).to.be('green');
 
-  expect(body.process.mem.external_bytes).to.be.a('number');
   expect(body.process.mem.heap_max_bytes).to.be.a('number');
   expect(body.process.mem.heap_used_bytes).to.be.a('number');
   expect(body.process.mem.resident_set_size_bytes).to.be.a('number');
   expect(body.process.pid).to.be.a('number');
   expect(body.process.uptime_ms).to.be.a('number');
 
-  expect(body.os.cpu.load_average['1m']).to.be.a('number');
   expect(body.os.mem.free_bytes).to.be.a('number');
   expect(body.os.mem.total_bytes).to.be.a('number');
   expect(body.os.uptime_ms).to.be.a('number');
 
+  expect(body.os.cpu.load_average['1m']).to.be.a('number');
+
   expect(body.response_times.avg_ms).not.to.be(null); // ok if is undefined
   expect(body.response_times.max_ms).not.to.be(null); // ok if is undefined
 

From 81f502bb5088111472230a97fb4759a67761378c Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 16:52:33 -0700
Subject: [PATCH 18/42] comment correction

---
 src/server/status/lib/source_kibana.js | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/server/status/lib/source_kibana.js b/src/server/status/lib/source_kibana.js
index b4d12dde46419..269ca06740820 100644
--- a/src/server/status/lib/source_kibana.js
+++ b/src/server/status/lib/source_kibana.js
@@ -22,10 +22,7 @@ import { get } from 'lodash';
 const snapshotRegex = /-snapshot/i;
 
 /**
- * This provides a common structure to apply to all Kibana monitoring documents so that they can be commonly
- * searched, field-collapsed, and aggregated against.
- *
- * 'sourceKibana' is akin to the `source_node` details in Elasticsearch nodes.
+ * This provides a meta data attribute along with Kibana stats.
  *
  * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core
  * @param {Object} config Server config

From 8130311dbb3905c47c3448ff7974f2f3d9bd1f52 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 16:59:51 -0700
Subject: [PATCH 19/42] reduce more loc change

---
 x-pack/plugins/monitoring/init.js                |  4 ++--
 .../server/kibana_monitoring/bulk_uploader.js    | 16 ++++++++--------
 .../monitoring/server/kibana_monitoring/init.js  |  2 +-
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/x-pack/plugins/monitoring/init.js b/x-pack/plugins/monitoring/init.js
index 1769cb7da4bb3..84c23c3774b6f 100644
--- a/x-pack/plugins/monitoring/init.js
+++ b/x-pack/plugins/monitoring/init.js
@@ -32,9 +32,9 @@ export const init = (monitoringPlugin, server) => {
   /*
    * Register collector objects for stats to show up in the APIs
    */
-  collectorSet.register(getOpsStatsCollector(server, kbnServer)); // TODO move to OSS probably
-  collectorSet.register(getSettingsCollector(server, kbnServer));
+  collectorSet.register(getOpsStatsCollector(server, kbnServer));
   collectorSet.register(getKibanaUsageCollector(server));
+  collectorSet.register(getSettingsCollector(server, kbnServer));
 
   /*
    * Instantiate and start the internal background task that calls collector
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
index 2c7e4520f80a2..1672f38ff9c1d 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
@@ -25,7 +25,7 @@ const LOGGING_TAGS = [LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG];
  * Handles internal Kibana stats collection and uploading data to Monitoring
  * bulk endpoint.
  *
- * TODO: remove this in 7.0
+ * NOTE: internal collection will be removed in 7.0
  *
  * Depends on
  *   - 'xpack.monitoring.kibana.collection.enabled' config
@@ -92,22 +92,18 @@ export class BulkUploader {
     this.stop('Connection issue detected');
   }
 
-  _onPayload(payload) {
-    return sendBulkPayload(this._client, this._interval, payload); // why is interval here?
-  }
-
   /*
    * @param {CollectorSet} collectorSet
    * @return {Promise} - resolves to undefined
    */
   async _fetchAndUpload(collectorSet) {
     const data = await collectorSet.bulkFetch(this._callClusterWithInternalUser);
-    const uploadData = BulkUploader.toBulkUploadFormat(data);
+    const payload = BulkUploader.toBulkUploadFormat(data);
 
-    if (uploadData) {
+    if (payload) {
       try {
         this._log.debug(`Uploading bulk stats payload to the local cluster`);
-        this._onPayload(uploadData);
+        this._onPayload(payload);
       } catch (err) {
         this._log.warn(err.stack);
         this._log.warn(`Unable to bulk upload the stats payload to the local cluster`);
@@ -117,6 +113,10 @@ export class BulkUploader {
     }
   }
 
+  _onPayload(payload) {
+    return sendBulkPayload(this._client, this._interval, payload);
+  }
+
   /*
    * Bulk stats are transformed into a bulk upload format
    * Non-legacy transformation is done in CollectorSet.toApiStats
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js
index 3e92d4d7c959d..6dcb0ae1fc076 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js
@@ -9,7 +9,7 @@ import { BulkUploader } from './bulk_uploader';
 /**
  * Initialize different types of Kibana Monitoring
  * TODO: remove this in 7.0
- * - Ops Events - essentially Kibana's /api/status (non-rolled up)
+ * - Ops Events - essentially Kibana's /api/status
  * - Usage Stats - essentially Kibana's /api/stats
  * - Kibana Settings - select uiSettings
  * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core

From 424352360981c0a4a35f81b323c1b2011d468d84 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 17:17:40 -0700
Subject: [PATCH 20/42] whitespace

---
 src/server/status/index.js                             |  1 -
 src/server/status/lib/metrics.test.js                  |  2 --
 .../server/kibana_monitoring/bulk_uploader.js          | 10 ++--------
 3 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/src/server/status/index.js b/src/server/status/index.js
index cc332b8ffebc7..aec9c9f1260df 100644
--- a/src/server/status/index.js
+++ b/src/server/status/index.js
@@ -26,7 +26,6 @@ export function statusMixin(kbnServer, server, config) {
   kbnServer.status = new ServerStatus(kbnServer.server);
 
   const statsCollector = getOpsStatsCollector(server, kbnServer);
-
   const { collectorSet } = server.usage;
   collectorSet.register(statsCollector);
 
diff --git a/src/server/status/lib/metrics.test.js b/src/server/status/lib/metrics.test.js
index 14c2d4932909a..087696ecfee4d 100644
--- a/src/server/status/lib/metrics.test.js
+++ b/src/server/status/lib/metrics.test.js
@@ -155,7 +155,6 @@ describe('Metrics', function () {
           }
         }
       });
-
     });
 
     it('parses event with missing fields / NaN for responseTimes.avg', () => {
@@ -176,7 +175,6 @@ describe('Metrics', function () {
         response_times: { max_in_millis: 4 },
         requests: { total: 22, disconnects: 0, status_codes: { '200': 22 } },
       });
-
     });
   });
 
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
index 1672f38ff9c1d..ae11c5c36eb02 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
@@ -144,10 +144,7 @@ export class BulkUploader {
       const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE);
       const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null;
       const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null;
-      statsResult = [
-        statsHeader,
-        statsPayload
-      ];
+      statsResult = [ statsHeader, statsPayload ];
       if (kibanaUsage) {
         set(statsResult, '[1].usage', kibanaUsage);
       }
@@ -160,10 +157,7 @@ export class BulkUploader {
     let settingsResult;
     const [ settingsHeader, settingsPayload ] = findItem(KIBANA_SETTINGS_TYPE);
     if (settingsHeader && settingsPayload) {
-      settingsResult = [
-        settingsHeader,
-        settingsPayload,
-      ];
+      settingsResult = [ settingsHeader, settingsPayload ];
     }
 
     // return new payload with the combined data

From 366aae9b91646fc371cef49f8f9f586b8e8b6a43 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Thu, 12 Jul 2018 17:18:26 -0700
Subject: [PATCH 21/42] comment question

---
 test/api_integration/apis/stats/stats.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/api_integration/apis/stats/stats.js b/test/api_integration/apis/stats/stats.js
index 0d0f8169d119f..ce947fde198d7 100644
--- a/test/api_integration/apis/stats/stats.js
+++ b/test/api_integration/apis/stats/stats.js
@@ -56,6 +56,7 @@ const assertStatsAndMetrics = body => {
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
+  // TODO possible to use supertestWithoutAuth service as well?
 
   describe('kibana stats api', () => {
     describe('basic', () => {

From 01d3bd8d7e008cdc10aa3c0f6d67d4c2418bbb42 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Fri, 13 Jul 2018 09:17:36 -0700
Subject: [PATCH 22/42] fix cluster_uuid

---
 src/server/status/routes/api/register_stats.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js
index 504c0ca0b7b9c..9b6f992c81d1e 100644
--- a/src/server/status/routes/api/register_stats.js
+++ b/src/server/status/routes/api/register_stats.js
@@ -68,7 +68,7 @@ export function registerStatsApi(kbnServer, server, config) {
             getUsage(callCluster),
             getClusterUuid(callCluster),
           ]);
-          extended = { usage, clusterUuid };
+          extended = { usage, cluster_uuid: clusterUuid };
         }
 
         /* kibana_stats gets singled out from the collector set as it is used

From d5c7e64b915a2fa98afe9e413d446d747cebbeb0 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Fri, 13 Jul 2018 09:57:39 -0700
Subject: [PATCH 23/42] fix stats integration test

---
 test/api_integration/apis/stats/stats.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/api_integration/apis/stats/stats.js b/test/api_integration/apis/stats/stats.js
index ce947fde198d7..2889255889fe9 100644
--- a/test/api_integration/apis/stats/stats.js
+++ b/test/api_integration/apis/stats/stats.js
@@ -91,7 +91,7 @@ export default function ({ getService }) {
           .expect(200)
           .then(({ body }) => {
             expect(body.cluster_uuid).to.be.a('string');
-            expect(body.usage.kibana).to.be.an('object');
+            expect(body.usage).to.be.an('object'); // no usage collectors have been registered so usage is an empty object
             assertStatsAndMetrics(body);
           });
       });
@@ -103,7 +103,7 @@ export default function ({ getService }) {
           .expect(200)
           .then(({ body }) => {
             expect(body.cluster_uuid).to.be.a('string');
-            expect(body.usage.kibana).to.be.an('object');
+            expect(body.usage).to.be.an('object');
             assertStatsAndMetrics(body);
           });
       });

From 23a40c5d001519b355a5bd9b5b8fbbca98b4cf9d Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Fri, 13 Jul 2018 09:46:40 -0700
Subject: [PATCH 24/42] fix bulk uploader test, combineTypes is no longer
 external

---
 .../__tests__/bulk_uploader.js                  | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js
index 17ebb46115605..0b69370b8e4c1 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js
@@ -52,8 +52,7 @@ describe('BulkUploader', () => {
       ]);
 
       const uploader = new BulkUploader(server, {
-        interval: FETCH_INTERVAL,
-        combineTypes: noop,
+        interval: FETCH_INTERVAL
       });
 
       uploader.start(collectors);
@@ -82,16 +81,11 @@ describe('BulkUploader', () => {
     });
 
     it('should run the bulk upload handler', done => {
-      const combineTypes = sinon.spy(data => {
-        return [data[0][0], { ...data[0][1], combined: true }];
-      });
-
       const collectors = new MockCollectorSet(server, [
         { fetch: () => ({ type: 'type_collector_test', result: { testData: 12345 } }) }
       ]);
       const uploader = new BulkUploader(server, {
-        interval: FETCH_INTERVAL,
-        combineTypes,
+        interval: FETCH_INTERVAL
       });
 
       uploader.start(collectors);
@@ -111,13 +105,6 @@ describe('BulkUploader', () => {
           'Uploading bulk stats payload to the local cluster',
         ]);
 
-        // un-flattened
-        const combineCalls = combineTypes.getCalls();
-        expect(combineCalls.length).to.be.greaterThan(0); // should be 1-2 fetch and combine cycles
-        expect(combineCalls[0].args).to.eql([
-          [[{ index: { _type: 'type_collector_test' } }, { testData: 12345 }]],
-        ]);
-
         done();
       }, CHECK_DELAY);
     });

From 07e6a671f611cb49f29ef008279c09f320998e9c Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Fri, 13 Jul 2018 10:23:08 -0700
Subject: [PATCH 25/42] very important comments about the current nature of
 stats represented and long-term goals

---
 .../status/collectors/get_ops_stats_collector.js    | 13 ++++++++++++-
 src/server/status/index.js                          |  2 +-
 src/server/status/routes/api/register_stats.js      |  3 +--
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/server/status/collectors/get_ops_stats_collector.js b/src/server/status/collectors/get_ops_stats_collector.js
index 985138872bc1f..15106e003765a 100644
--- a/src/server/status/collectors/get_ops_stats_collector.js
+++ b/src/server/status/collectors/get_ops_stats_collector.js
@@ -23,6 +23,17 @@ import { sourceKibana } from '../lib';
 
 /*
  * Initialize a collector for Kibana Ops Stats
+ *
+ * NOTE this collector's fetch method returns the latest stats from the
+ * Hapi/Good/Even-Better ops event listener. Therefore, the stats reset
+ * every 5 seconds (the default value of the ops.interval configuration
+ * setting). That makes it geared for providing the latest "real-time"
+ * stats. In the long-term, fetch should return stats that constantly
+ * accumulate over the server's uptime for better machine readability.
+ * Since the data is captured, timestamped and stored, the historical
+ * data can provide "real-time" stats by calculating a derivative of
+ * the metrics.
+ * See PR comment in https://github.com/elastic/kibana/pull/20577/files#r202416647
  */
 export function getOpsStatsCollector(server, kbnServer) {
   const { collectorSet } = server.usage;
@@ -31,7 +42,7 @@ export function getOpsStatsCollector(server, kbnServer) {
     fetch: () => {
       return {
         kibana: sourceKibana(server, kbnServer),
-        ...kbnServer.metrics
+        ...kbnServer.metrics // latest metrics captured from the ops event listener in src/server/status/index
       };
     }
   });
diff --git a/src/server/status/index.js b/src/server/status/index.js
index aec9c9f1260df..f05a14e3367af 100644
--- a/src/server/status/index.js
+++ b/src/server/status/index.js
@@ -35,7 +35,7 @@ export function statusMixin(kbnServer, server, config) {
     const metrics = new Metrics(config, server);
 
     evenBetter.monitor.on('ops', event => {
-      metrics.capture(event).then(data => { kbnServer.metrics = data; });
+      metrics.capture(event).then(data => { kbnServer.metrics = data; }); // captures (performs transforms on) the latest event data and stashes the metrics for status/stats API payload
     });
   }
 
diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js
index 9b6f992c81d1e..e95258d7c4a07 100644
--- a/src/server/status/routes/api/register_stats.js
+++ b/src/server/status/routes/api/register_stats.js
@@ -73,8 +73,7 @@ export function registerStatsApi(kbnServer, server, config) {
 
         /* kibana_stats gets singled out from the collector set as it is used
          * for health-checking Kibana and fetch does not rely on fetching data
-         * from ES
-         */
+         * from ES */
         const kibanaStatsCollector = collectorSet.getCollectorByType(KIBANA_STATS_TYPE);
         let kibanaStats = await kibanaStatsCollector.fetch();
         kibanaStats = collectorSet.toApiFieldNames(kibanaStats);

From c11c68dc9e408d00a8cd0dfe1e857df3a5a90b5b Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Fri, 13 Jul 2018 11:26:33 -0700
Subject: [PATCH 26/42] add stats api tests with/without authentication

---
 .../status/routes/api/register_stats.js       | 15 +++--
 test/api_integration/apis/stats/stats.js      |  2 -
 x-pack/test/api_integration/apis/index.js     |  1 +
 .../test/api_integration/apis/kibana/index.js | 11 ++++
 .../apis/kibana/stats/index.js                | 11 ++++
 .../apis/kibana/stats/stats.js                | 62 +++++++++++++++++++
 6 files changed, 95 insertions(+), 7 deletions(-)
 create mode 100644 x-pack/test/api_integration/apis/kibana/index.js
 create mode 100644 x-pack/test/api_integration/apis/kibana/stats/index.js
 create mode 100644 x-pack/test/api_integration/apis/kibana/stats/stats.js

diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js
index e95258d7c4a07..abbe4f0a95ff9 100644
--- a/src/server/status/routes/api/register_stats.js
+++ b/src/server/status/routes/api/register_stats.js
@@ -18,6 +18,7 @@
  */
 
 import Joi from 'joi';
+import { boomify } from 'boom';
 import { wrapAuthConfig } from '../../wrap_auth_config';
 import { KIBANA_STATS_TYPE } from '../../constants';
 
@@ -64,11 +65,15 @@ export function registerStatsApi(kbnServer, server, config) {
         if (isExtended) {
           const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('admin');
           const callCluster = (...args) => callWithRequest(req, ...args);
-          const [ usage, clusterUuid ] = await Promise.all([
-            getUsage(callCluster),
-            getClusterUuid(callCluster),
-          ]);
-          extended = { usage, cluster_uuid: clusterUuid };
+          try {
+            const [ usage, clusterUuid ] = await Promise.all([
+              getUsage(callCluster),
+              getClusterUuid(callCluster),
+            ]);
+            extended = { usage, cluster_uuid: clusterUuid };
+          } catch (e) {
+            return reply(boomify(e));
+          }
         }
 
         /* kibana_stats gets singled out from the collector set as it is used
diff --git a/test/api_integration/apis/stats/stats.js b/test/api_integration/apis/stats/stats.js
index 2889255889fe9..a708af7e02198 100644
--- a/test/api_integration/apis/stats/stats.js
+++ b/test/api_integration/apis/stats/stats.js
@@ -56,8 +56,6 @@ const assertStatsAndMetrics = body => {
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
-  // TODO possible to use supertestWithoutAuth service as well?
-
   describe('kibana stats api', () => {
     describe('basic', () => {
       it('should return the stats without cluster_uuid with no query string params', () => {
diff --git a/x-pack/test/api_integration/apis/index.js b/x-pack/test/api_integration/apis/index.js
index 4068aa307b08a..7f105650141d9 100644
--- a/x-pack/test/api_integration/apis/index.js
+++ b/x-pack/test/api_integration/apis/index.js
@@ -10,5 +10,6 @@ export default function ({ loadTestFile }) {
     loadTestFile(require.resolve('./monitoring'));
     loadTestFile(require.resolve('./xpack_main'));
     loadTestFile(require.resolve('./logstash'));
+    loadTestFile(require.resolve('./kibana'));
   });
 }
diff --git a/x-pack/test/api_integration/apis/kibana/index.js b/x-pack/test/api_integration/apis/kibana/index.js
new file mode 100644
index 0000000000000..d84ac9c75154a
--- /dev/null
+++ b/x-pack/test/api_integration/apis/kibana/index.js
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export default function ({ loadTestFile }) {
+  describe('kibana', () => {
+    loadTestFile(require.resolve('./stats'));
+  });
+}
diff --git a/x-pack/test/api_integration/apis/kibana/stats/index.js b/x-pack/test/api_integration/apis/kibana/stats/index.js
new file mode 100644
index 0000000000000..265a64dad7d13
--- /dev/null
+++ b/x-pack/test/api_integration/apis/kibana/stats/index.js
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export default function ({ loadTestFile }) {
+  describe('stats', () => {
+    loadTestFile(require.resolve('./stats'));
+  });
+}
diff --git a/x-pack/test/api_integration/apis/kibana/stats/stats.js b/x-pack/test/api_integration/apis/kibana/stats/stats.js
new file mode 100644
index 0000000000000..9f71df9071ebf
--- /dev/null
+++ b/x-pack/test/api_integration/apis/kibana/stats/stats.js
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from 'expect.js';
+
+export default function ({ getService }) {
+  const supertestNoAuth = getService('supertestWithoutAuth');
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('/api/stats', () => {
+    describe('operational stats and usage stats', () => {
+      before('load clusters archive', () => {
+        return esArchiver.load('discover');
+      });
+
+      after('unload clusters archive', () => {
+        return esArchiver.unload('discover');
+      });
+
+      describe('no auth', () => {
+        it('should return 200 and stats for no extended', async () => {
+          const { body } = await supertestNoAuth
+            .get('/api/stats')
+            .expect(200);
+          expect(body.kibana.uuid).to.eql('5b2de169-2785-441b-ae8c-186a1936b17d');
+          expect(body.uptime_ms).to.be.greaterThan(0);
+          expect(body.usage).to.be(undefined);
+        });
+
+        it('should return 401 for extended', async () => {
+          await supertestNoAuth
+            .get('/api/stats?extended')
+            .expect(401); // unauthorized
+        });
+      });
+
+      describe('with auth', () => {
+        it('should return 200 and stats for no extended', async () => {
+          const { body } = await supertest
+            .get('/api/stats')
+            .expect(200);
+          expect(body.kibana.uuid).to.eql('5b2de169-2785-441b-ae8c-186a1936b17d');
+          expect(body.uptime_ms).to.be.greaterThan(0);
+        });
+
+        it('should return 200 for extended', async () => {
+          const { body } = await supertest
+            .get('/api/stats?extended')
+            .expect(200);
+          expect(body.kibana.uuid).to.eql('5b2de169-2785-441b-ae8c-186a1936b17d');
+          expect(body.uptime_ms).to.be.greaterThan(0);
+          expect(body.usage.kibana.index).to.be('.kibana');
+          expect(body.usage.kibana.dashboard.total).to.be(0);
+        });
+      });
+    });
+  });
+}

From 7a149e04ab8959ea3125003beb0503eca9038936 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Fri, 13 Jul 2018 12:06:40 -0700
Subject: [PATCH 27/42] fix more fields to match data model

---
 src/server/status/lib/metrics.js           | 44 ++++++++++++----------
 test/api_integration/apis/stats/stats.js   | 17 +++++----
 test/api_integration/apis/status/status.js | 11 +++---
 3 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/src/server/status/lib/metrics.js b/src/server/status/lib/metrics.js
index ec3c115de1705..88d12bd854d73 100644
--- a/src/server/status/lib/metrics.js
+++ b/src/server/status/lib/metrics.js
@@ -18,6 +18,7 @@
  */
 
 import os from 'os';
+import v8 from 'v8';
 import { get, isObject, merge } from 'lodash';
 import { keysToSnakeCaseShallow } from '../../../utils/case_conversion';
 import { getAllStats as cGroupStats } from './cgroup';
@@ -32,11 +33,13 @@ export class Metrics {
   static getStubMetrics() {
     return {
       process: {
-        mem: {}
+        memory: {
+          heap: {}
+        }
       },
       os: {
         cpu: {},
-        mem: {}
+        memory: {}
       },
       response_times: {},
       requests: {
@@ -56,7 +59,6 @@ export class Metrics {
 
     const metrics = {
       last_updated: timestamp,
-      uptime_in_millis: event.process.uptime_ms, // TODO: deprecate this field, data should only have process.uptime_ms
       collection_interval_in_millis: this.config.get('ops.interval')
     };
 
@@ -64,35 +66,38 @@ export class Metrics {
   }
 
   captureEvent(hapiEvent) {
+    const heapStats = v8.getHeapStatistics();
     const port = this.config.get('server.port');
-
     const avgInMillis = get(hapiEvent, ['responseTimes', port, 'avg']); // sadly, it's possible for this to be NaN
     const maxInMillis = get(hapiEvent, ['responseTimes', port, 'max']);
 
     return {
       process: {
-        mem: {
-          // https://nodejs.org/docs/latest-v8.x/api/process.html#process_process_memoryusage
-          heap_max_in_bytes: get(hapiEvent, 'psmem.heapTotal'),
-          heap_used_in_bytes: get(hapiEvent, 'psmem.heapUsed'),
+        memory: {
+          heap: {
+            // https://nodejs.org/docs/latest-v8.x/api/process.html#process_process_memoryusage
+            total_in_bytes: get(hapiEvent, 'psmem.heapTotal'),
+            used_in_bytes: get(hapiEvent, 'psmem.heapUsed'),
+            size_limit: heapStats.heap_size_limit
+          },
           resident_set_size_in_bytes: get(hapiEvent, 'psmem.rss'),
         },
+        event_loop_delay: get(hapiEvent, 'psdelay'), // TODO: should be event_loop_delay_ms
         pid: process.pid,
-        uptime_ms: process.uptime() * 1000
+        uptime_in_millis: process.uptime() * 1000
       },
       os: {
-        cpu: {
-          load_average: {
-            '1m': get(hapiEvent, 'osload.0'),
-            '5m': get(hapiEvent, 'osload.1'),
-            '15m': get(hapiEvent, 'osload.2')
-          }
+        load: {
+          '1m': get(hapiEvent, 'osload.0'),
+          '5m': get(hapiEvent, 'osload.1'),
+          '15m': get(hapiEvent, 'osload.2')
         },
-        mem: {
+        memory: {
+          total_in_bytes: os.totalmem(),
           free_in_bytes: os.freemem(),
-          total_in_bytes: os.totalmem()
+          used_in_bytes: get(hapiEvent, 'osmem.total') - get(hapiEvent, 'osmem.free')
         },
-        uptime_ms: os.uptime() * 1000
+        uptime_in_millis: os.uptime() * 1000
       },
       response_times: {
         avg_in_millis: isNaN(avgInMillis) ? undefined : avgInMillis, // convert NaN to undefined
@@ -100,8 +105,7 @@ export class Metrics {
       },
       requests: keysToSnakeCaseShallow(get(hapiEvent, ['requests', port])),
       concurrent_connections: get(hapiEvent, ['concurrents', port]),
-      sockets: get(hapiEvent, 'sockets'),
-      event_loop_delay: get(hapiEvent, 'psdelay')
+      sockets: get(hapiEvent, 'sockets')
     };
   }
 
diff --git a/test/api_integration/apis/stats/stats.js b/test/api_integration/apis/stats/stats.js
index a708af7e02198..0dbd8f4d61641 100644
--- a/test/api_integration/apis/stats/stats.js
+++ b/test/api_integration/apis/stats/stats.js
@@ -28,17 +28,21 @@ const assertStatsAndMetrics = body => {
   expect(body.kibana.snapshot).to.be.a('boolean');
   expect(body.kibana.status).to.be('green');
 
-  expect(body.process.mem.heap_max_bytes).to.be.a('number');
-  expect(body.process.mem.heap_used_bytes).to.be.a('number');
-  expect(body.process.mem.resident_set_size_bytes).to.be.a('number');
+  expect(body.process.memory.heap.total_bytes).to.be.a('number');
+  expect(body.process.memory.heap.used_bytes).to.be.a('number');
+  expect(body.process.memory.heap.size_limit).to.be.a('number');
+  expect(body.process.memory.resident_set_size_bytes).to.be.a('number');
   expect(body.process.pid).to.be.a('number');
   expect(body.process.uptime_ms).to.be.a('number');
+  expect(body.process.event_loop_delay).to.be.a('number');
 
-  expect(body.os.mem.free_bytes).to.be.a('number');
-  expect(body.os.mem.total_bytes).to.be.a('number');
+  expect(body.os.memory.free_bytes).to.be.a('number');
+  expect(body.os.memory.total_bytes).to.be.a('number');
   expect(body.os.uptime_ms).to.be.a('number');
 
-  expect(body.os.cpu.load_average['1m']).to.be.a('number');
+  expect(body.os.load['1m']).to.be.a('number');
+  expect(body.os.load['5m']).to.be.a('number');
+  expect(body.os.load['15m']).to.be.a('number');
 
   expect(body.response_times.avg_ms).not.to.be(null); // ok if is undefined
   expect(body.response_times.max_ms).not.to.be(null); // ok if is undefined
@@ -51,7 +55,6 @@ const assertStatsAndMetrics = body => {
   expect(body.sockets.https.total).to.be.a('number');
 
   expect(body.concurrent_connections).to.be.a('number');
-  expect(body.event_loop_delay).to.be.a('number');
 };
 
 export default function ({ getService }) {
diff --git a/test/api_integration/apis/status/status.js b/test/api_integration/apis/status/status.js
index a64bdc8ebf862..969558d38f93b 100644
--- a/test/api_integration/apis/status/status.js
+++ b/test/api_integration/apis/status/status.js
@@ -46,12 +46,13 @@ export default function ({ getService }) {
 
           expect(body.metrics.collection_interval_in_millis).to.be.a('number');
 
-          expect(body.metrics.process.mem.heap_max_in_bytes).to.be.a('number');
-          expect(body.metrics.process.mem.heap_used_in_bytes).to.be.a('number');
+          expect(body.metrics.process.memory.heap.total_in_bytes).to.be.a('number');
+          expect(body.metrics.process.memory.heap.used_in_bytes).to.be.a('number');
+          expect(body.metrics.process.memory.heap.size_limit).to.be.a('number');
 
-          expect(body.metrics.os.cpu.load_average['1m']).to.be.a('number');
-          expect(body.metrics.os.cpu.load_average['5m']).to.be.a('number');
-          expect(body.metrics.os.cpu.load_average['15m']).to.be.a('number');
+          expect(body.metrics.os.load['1m']).to.be.a('number');
+          expect(body.metrics.os.load['5m']).to.be.a('number');
+          expect(body.metrics.os.load['15m']).to.be.a('number');
 
           expect(body.metrics.response_times.avg_in_millis).not.to.be(null); // ok if undefined
           expect(body.metrics.response_times.max_in_millis).not.to.be(null); // ok if undefined

From 82f4283c3a45d13d33b56f1ee30f8696c146c509 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Fri, 13 Jul 2018 13:06:11 -0700
Subject: [PATCH 28/42] fix more tests

---
 x-pack/test/api_integration/apis/kibana/stats/stats.js | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/x-pack/test/api_integration/apis/kibana/stats/stats.js b/x-pack/test/api_integration/apis/kibana/stats/stats.js
index 9f71df9071ebf..e4a7c96328252 100644
--- a/x-pack/test/api_integration/apis/kibana/stats/stats.js
+++ b/x-pack/test/api_integration/apis/kibana/stats/stats.js
@@ -27,7 +27,8 @@ export default function ({ getService }) {
             .get('/api/stats')
             .expect(200);
           expect(body.kibana.uuid).to.eql('5b2de169-2785-441b-ae8c-186a1936b17d');
-          expect(body.uptime_ms).to.be.greaterThan(0);
+          expect(body.process.uptime_ms).to.be.greaterThan(0);
+          expect(body.os.uptime_ms).to.be.greaterThan(0);
           expect(body.usage).to.be(undefined);
         });
 
@@ -44,7 +45,8 @@ export default function ({ getService }) {
             .get('/api/stats')
             .expect(200);
           expect(body.kibana.uuid).to.eql('5b2de169-2785-441b-ae8c-186a1936b17d');
-          expect(body.uptime_ms).to.be.greaterThan(0);
+          expect(body.process.uptime_ms).to.be.greaterThan(0);
+          expect(body.os.uptime_ms).to.be.greaterThan(0);
         });
 
         it('should return 200 for extended', async () => {
@@ -52,7 +54,8 @@ export default function ({ getService }) {
             .get('/api/stats?extended')
             .expect(200);
           expect(body.kibana.uuid).to.eql('5b2de169-2785-441b-ae8c-186a1936b17d');
-          expect(body.uptime_ms).to.be.greaterThan(0);
+          expect(body.process.uptime_ms).to.be.greaterThan(0);
+          expect(body.os.uptime_ms).to.be.greaterThan(0);
           expect(body.usage.kibana.index).to.be('.kibana');
           expect(body.usage.kibana.dashboard.total).to.be(0);
         });

From 1c4cc37eb59b6700f5dd0383887b2f305f009c9e Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Fri, 13 Jul 2018 15:13:03 -0700
Subject: [PATCH 29/42] fix jest test

---
 src/server/status/lib/metrics.test.js | 32 +++++++++++++--------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/src/server/status/lib/metrics.test.js b/src/server/status/lib/metrics.test.js
index 087696ecfee4d..e4b0fa0c39a5d 100644
--- a/src/server/status/lib/metrics.test.js
+++ b/src/server/status/lib/metrics.test.js
@@ -75,7 +75,6 @@ describe('Metrics', function () {
       expect(capturedMetrics).toMatchObject({
         last_updated: '2017-04-14T18:35:41.534Z',
         collection_interval_in_millis: 5000,
-        uptime_in_millis: 1980,
         a: [ { b: 2, c: 3 }, { d: 4, e: 5 } ], process: { uptime_ms: 1980 }
       });
     });
@@ -112,28 +111,27 @@ describe('Metrics', function () {
 
       expect(metrics.captureEvent(hapiEvent)).toMatchObject({
         'concurrent_connections': 0,
-        'event_loop_delay': 1.6091690063476562,
         'os': {
-          'cpu': {
-            'load_average': {
-              '15m': 1.89794921875,
-              '1m': 2.20751953125,
-              '5m': 2.02294921875
-            }
+          'load': {
+            '15m': 1.89794921875,
+            '1m': 2.20751953125,
+            '5m': 2.02294921875
           },
-          'mem': {
+          'memory': {
             'free_in_bytes': 12,
             'total_in_bytes': 24,
           },
+          'uptime_in_millis': 12000000,
         },
         'process': {
-          'mem': {
-            'heap_max_in_bytes': 168194048,
-            'heap_used_in_bytes': 130553400,
+          'memory': {
+            'heap': {
+              'total_in_bytes': 168194048,
+              'used_in_bytes': 130553400,
+            },
             'resident_set_size_in_bytes': 193716224,
           },
-          'pid': 8675309,
-          'uptime_ms': 5000000
+          'pid': 8675309
         },
         'requests': {
           'disconnects': 0,
@@ -167,10 +165,10 @@ describe('Metrics', function () {
       };
 
       expect(metrics.captureEvent(hapiEvent)).toMatchObject({
-        process: { mem: {}, pid: 8675309, uptime_ms: 5000000 },
+        process: { memory: { heap: {} }, pid: 8675309, uptime_in_millis: 5000000 },
         os: {
-          cpu: { load_average: {} },
-          mem: { free_in_bytes: 12, total_in_bytes: 24 },
+          load: {},
+          memory: { free_in_bytes: 12, total_in_bytes: 24 },
         },
         response_times: { max_in_millis: 4 },
         requests: { total: 22, disconnects: 0, status_codes: { '200': 22 } },

From 7366cb6557ad12281f7f2e1f0ca31f644f0e3165 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Mon, 16 Jul 2018 13:35:22 -0700
Subject: [PATCH 30/42] remove TODO

---
 src/server/status/lib/metrics.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server/status/lib/metrics.js b/src/server/status/lib/metrics.js
index 88d12bd854d73..293a64fddce4d 100644
--- a/src/server/status/lib/metrics.js
+++ b/src/server/status/lib/metrics.js
@@ -82,7 +82,7 @@ export class Metrics {
           },
           resident_set_size_in_bytes: get(hapiEvent, 'psmem.rss'),
         },
-        event_loop_delay: get(hapiEvent, 'psdelay'), // TODO: should be event_loop_delay_ms
+        event_loop_delay: get(hapiEvent, 'psdelay'),
         pid: process.pid,
         uptime_in_millis: process.uptime() * 1000
       },

From ffe3f8f61ec05ba54c4ad24f5b3e992abe5a44a9 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Mon, 16 Jul 2018 13:52:43 -0700
Subject: [PATCH 31/42] remove sockets

---
 src/server/status/lib/metrics.js                     |  5 -----
 src/server/status/lib/metrics.test.js                | 12 ------------
 .../collectors/ops_buffer/__tests__/event_roller.js  |  4 ----
 3 files changed, 21 deletions(-)

diff --git a/src/server/status/lib/metrics.js b/src/server/status/lib/metrics.js
index 293a64fddce4d..1a7bcce65dfd7 100644
--- a/src/server/status/lib/metrics.js
+++ b/src/server/status/lib/metrics.js
@@ -44,10 +44,6 @@ export class Metrics {
       response_times: {},
       requests: {
         status_codes: {}
-      },
-      sockets: {
-        http: {},
-        https: {}
       }
     };
   }
@@ -105,7 +101,6 @@ export class Metrics {
       },
       requests: keysToSnakeCaseShallow(get(hapiEvent, ['requests', port])),
       concurrent_connections: get(hapiEvent, ['concurrents', port]),
-      sockets: get(hapiEvent, 'sockets')
     };
   }
 
diff --git a/src/server/status/lib/metrics.test.js b/src/server/status/lib/metrics.test.js
index e4b0fa0c39a5d..c0afa65749548 100644
--- a/src/server/status/lib/metrics.test.js
+++ b/src/server/status/lib/metrics.test.js
@@ -95,10 +95,6 @@ describe('Metrics', function () {
       const hapiEvent = {
         'requests': { '5603': { 'total': 22, 'disconnects': 0, 'statusCodes': { '200': 22 } } },
         'responseTimes': { '5603': { 'avg': 1.8636363636363635, 'max': 4 } },
-        'sockets': {
-          'http': { 'total': 0 },
-          'https': { 'total': 0 }
-        },
         'osload': [2.20751953125, 2.02294921875, 1.89794921875],
         'osmem': { 'total': 17179869184, 'free': 102318080 },
         'osup': 1008991,
@@ -144,14 +140,6 @@ describe('Metrics', function () {
           'avg_in_millis': 1.8636363636363635,
           'max_in_millis': 4
         },
-        'sockets': {
-          'http': {
-            'total': 0
-          },
-          'https': {
-            'total': 0
-          }
-        }
       });
     });
 
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/__tests__/event_roller.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/__tests__/event_roller.js
index 55adc025ef8bd..bb4af020f4821 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/__tests__/event_roller.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/__tests__/event_roller.js
@@ -17,10 +17,6 @@ const events = [
       }
     },
     responseTimes: { '5601': { avg: 5.213592233009709, max: 36 } },
-    sockets: {
-      http: { total: 1, '169.254.169.254:80:': 1 },
-      https: { total: 0 }
-    },
     osload: [1.90380859375, 1.84033203125, 1.82666015625],
     osmem: { total: 17179869184, free: 613638144 },
     osup: 4615,

From ab67e641c93f20d2df59eb6bbb47de887ae4f5ac Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Mon, 16 Jul 2018 13:53:09 -0700
Subject: [PATCH 32/42] use snake_case for api field names

---
 src/server/status/routes/api/register_stats.js | 5 ++---
 src/server/usage/classes/collector_set.js      | 5 ++++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js
index abbe4f0a95ff9..8eb69e96d94c3 100644
--- a/src/server/status/routes/api/register_stats.js
+++ b/src/server/status/routes/api/register_stats.js
@@ -42,8 +42,7 @@ export function registerStatsApi(kbnServer, server, config) {
 
   const getUsage = async callCluster => {
     const usage = await collectorSet.bulkFetchUsage(callCluster);
-    const usageObject = collectorSet.toObject(usage);
-    return collectorSet.toApiFieldNames(usageObject);
+    return collectorSet.toObject(usage);
   };
 
   server.route(
@@ -70,7 +69,7 @@ export function registerStatsApi(kbnServer, server, config) {
               getUsage(callCluster),
               getClusterUuid(callCluster),
             ]);
-            extended = { usage, cluster_uuid: clusterUuid };
+            extended = collectorSet.toApiFieldNames({ usage, clusterUuid });
           } catch (e) {
             return reply(boomify(e));
           }
diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js
index 640c6a27f5faa..086bc65754ca2 100644
--- a/src/server/usage/classes/collector_set.js
+++ b/src/server/usage/classes/collector_set.js
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+import { snakeCase } from 'lodash';
 import Promise from 'bluebird';
 import { getCollectorLogger } from '../lib';
 import { Collector } from './collector';
@@ -120,7 +121,9 @@ export class CollectorSet {
       const value = apiData[currName];
 
       let newName = currName;
-      newName = newName.replace('_in_bytes', '_bytes');
+      newName = snakeCase(newName);
+      newName = newName.replace(/^(1|5|15)_m/, '$1m'); // os.load.15m, os.load.5m, os.load.1m
+      newName = newName.replace('_in_millis', '_ms');
       newName = newName.replace('_in_millis', '_ms');
 
       return {

From 64728d9e0ff39f9b3307c4ae0c99f6afb4a58d21 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Tue, 17 Jul 2018 09:58:32 -0700
Subject: [PATCH 33/42] restore accidental removal + copy/paste error

---
 src/server/usage/classes/collector_set.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js
index 086bc65754ca2..91834565bfab3 100644
--- a/src/server/usage/classes/collector_set.js
+++ b/src/server/usage/classes/collector_set.js
@@ -123,7 +123,7 @@ export class CollectorSet {
       let newName = currName;
       newName = snakeCase(newName);
       newName = newName.replace(/^(1|5|15)_m/, '$1m'); // os.load.15m, os.load.5m, os.load.1m
-      newName = newName.replace('_in_millis', '_ms');
+      newName = newName.replace('_in_bytes', '_bytes');
       newName = newName.replace('_in_millis', '_ms');
 
       return {

From fb6fca7c938edaadc95f9e577175ccb7679ca0cf Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Tue, 17 Jul 2018 10:06:30 -0700
Subject: [PATCH 34/42] sourceKibana -> getKibanaInfoForStats

---
 src/server/status/collectors/get_ops_stats_collector.js       | 4 ++--
 .../lib/{source_kibana.js => get_kibana_info_for_stats.js}    | 2 +-
 src/server/status/lib/index.js                                | 2 +-
 .../kibana_monitoring/collectors/get_ops_stats_collector.js   | 4 ++--
 .../kibana_monitoring/collectors/get_settings_collector.js    | 4 ++--
 .../lib/{source_kibana.js => get_kibana_info_for_stats.js}    | 4 +---
 .../plugins/monitoring/server/kibana_monitoring/lib/index.js  | 2 +-
 7 files changed, 10 insertions(+), 12 deletions(-)
 rename src/server/status/lib/{source_kibana.js => get_kibana_info_for_stats.js} (96%)
 rename x-pack/plugins/monitoring/server/kibana_monitoring/lib/{source_kibana.js => get_kibana_info_for_stats.js} (90%)

diff --git a/src/server/status/collectors/get_ops_stats_collector.js b/src/server/status/collectors/get_ops_stats_collector.js
index 15106e003765a..ba4a3d92699ae 100644
--- a/src/server/status/collectors/get_ops_stats_collector.js
+++ b/src/server/status/collectors/get_ops_stats_collector.js
@@ -19,7 +19,7 @@
 
 
 import { KIBANA_STATS_TYPE } from '../constants';
-import { sourceKibana } from '../lib';
+import { getKibanaInfoForStats } from '../lib';
 
 /*
  * Initialize a collector for Kibana Ops Stats
@@ -41,7 +41,7 @@ export function getOpsStatsCollector(server, kbnServer) {
     type: KIBANA_STATS_TYPE,
     fetch: () => {
       return {
-        kibana: sourceKibana(server, kbnServer),
+        kibana: getKibanaInfoForStats(server, kbnServer),
         ...kbnServer.metrics // latest metrics captured from the ops event listener in src/server/status/index
       };
     }
diff --git a/src/server/status/lib/source_kibana.js b/src/server/status/lib/get_kibana_info_for_stats.js
similarity index 96%
rename from src/server/status/lib/source_kibana.js
rename to src/server/status/lib/get_kibana_info_for_stats.js
index 269ca06740820..bac800c03b2b1 100644
--- a/src/server/status/lib/source_kibana.js
+++ b/src/server/status/lib/get_kibana_info_for_stats.js
@@ -29,7 +29,7 @@ const snapshotRegex = /-snapshot/i;
  * @param {String} host Kibana host
  * @return {Object} The object containing a "kibana" field and source instance details.
  */
-export function sourceKibana(server, kbnServer) {
+export function getKibanaInfoForStats(server, kbnServer) {
   const config = server.config();
   const status = kbnServer.status.toJSON();
 
diff --git a/src/server/status/lib/index.js b/src/server/status/lib/index.js
index 27f7f730d6cce..93db8b2d22561 100644
--- a/src/server/status/lib/index.js
+++ b/src/server/status/lib/index.js
@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export { sourceKibana } from './source_kibana';
+export { getKibanaInfoForStats } from './get_kibana_info_for_stats';
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
index 9e551e89480a0..616d8afc3441f 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
@@ -6,7 +6,7 @@
 
 import { KIBANA_STATS_TYPE } from '../../../common/constants';
 import { opsBuffer } from './ops_buffer';
-import { sourceKibana } from '../lib';
+import { getKibanaInfoForStats } from '../lib';
 
 /*
  * Initialize a collector for Kibana Ops Stats
@@ -49,7 +49,7 @@ export function getOpsStatsCollector(server, kbnServer) {
     init: start,
     fetch: () => {
       return {
-        kibana: sourceKibana(server, kbnServer),
+        kibana: getKibanaInfoForStats(server, kbnServer),
         ...buffer.flush()
       };
     }
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js
index d9913b691e97f..1382abb4ea293 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js
@@ -7,7 +7,7 @@
 import { get } from 'lodash';
 import { XPACK_DEFAULT_ADMIN_EMAIL_UI_SETTING } from '../../../../../server/lib/constants';
 import { KIBANA_SETTINGS_TYPE } from '../../../common/constants';
-import { sourceKibana } from '../lib';
+import { getKibanaInfoForStats } from '../lib';
 
 /*
  * Check if Cluster Alert email notifications is enabled in config
@@ -80,7 +80,7 @@ export function getSettingsCollector(server, kbnServer) {
       shouldUseNull = !!defaultAdminEmail;
 
       return {
-        kibana: sourceKibana(server, kbnServer),
+        kibana: getKibanaInfoForStats(server, kbnServer),
         ...kibanaSettingsData
       };
     }
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_kibana_info_for_stats.js
similarity index 90%
rename from x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js
rename to x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_kibana_info_for_stats.js
index b965c526d2323..7c9cb7dc530c9 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/source_kibana.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/get_kibana_info_for_stats.js
@@ -12,14 +12,12 @@ const snapshotRegex = /-snapshot/i;
  * This provides a common structure to apply to all Kibana monitoring documents so that they can be commonly
  * searched, field-collapsed, and aggregated against.
  *
- * 'sourceKibana' is akin to the `source_node` details in Elasticsearch nodes.
- *
  * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core
  * @param {Object} config Server config
  * @param {String} host Kibana host
  * @return {Object} The object containing a "kibana" field and source instance details.
  */
-export function sourceKibana(server, kbnServer) {
+export function getKibanaInfoForStats(server, kbnServer) {
   const config = server.config();
   const status = kbnServer.status.toJSON();
 
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js
index 0b68a0aed0d2f..56a2f48de88db 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/lib/index.js
@@ -6,4 +6,4 @@
 
 export { sendBulkPayload } from './send_bulk_payload';
 export { monitoringBulk } from './monitoring_bulk';
-export { sourceKibana } from './source_kibana';
+export { getKibanaInfoForStats } from './get_kibana_info_for_stats';

From b3c92e5f982ffc160ab2407a79c147899e8bb435 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Tue, 17 Jul 2018 10:20:26 -0700
Subject: [PATCH 35/42] skip usage test on legacy endpoint

---
 x-pack/test/reporting/api/usage.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/x-pack/test/reporting/api/usage.js b/x-pack/test/reporting/api/usage.js
index 018f40eef5118..3769d10796c70 100644
--- a/x-pack/test/reporting/api/usage.js
+++ b/x-pack/test/reporting/api/usage.js
@@ -124,7 +124,10 @@ export default function ({ getService }) {
       });
     });
 
-    describe('deprecated API', () => {
+    /* have to skip this test because the usage stats returned by the legacy
+     * endpoint aren't snake_cased in the legacy usage api
+     */
+    describe.skip('deprecated API', () => {
       it('shows correct stats', async () => {
         const usage = await usageAPI.getUsageStatsFromDeprecatedPre64Endpoint();
 

From 4ee99b95fde51c93fd08e3b3cc63b188f4598bbe Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Tue, 17 Jul 2018 12:49:08 -0700
Subject: [PATCH 36/42] fix api tests

---
 test/api_integration/apis/stats/stats.js        |  3 ---
 x-pack/test/reporting/services/reporting_api.js | 12 ++++++------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/test/api_integration/apis/stats/stats.js b/test/api_integration/apis/stats/stats.js
index 0dbd8f4d61641..4f25913b640c7 100644
--- a/test/api_integration/apis/stats/stats.js
+++ b/test/api_integration/apis/stats/stats.js
@@ -51,9 +51,6 @@ const assertStatsAndMetrics = body => {
   expect(body.requests.total).to.be.a('number');
   expect(body.requests.disconnects).to.be.a('number');
 
-  expect(body.sockets.http.total).to.be.a('number');
-  expect(body.sockets.https.total).to.be.a('number');
-
   expect(body.concurrent_connections).to.be.a('number');
 };
 
diff --git a/x-pack/test/reporting/services/reporting_api.js b/x-pack/test/reporting/services/reporting_api.js
index 361ed94c5066a..0c66a77a65fe2 100644
--- a/x-pack/test/reporting/services/reporting_api.js
+++ b/x-pack/test/reporting/services/reporting_api.js
@@ -98,8 +98,8 @@ export function ReportingAPIProvider({ getService }) {
     },
 
     expectRecentPdfAppStats(stats, app, count) {
-      expect(stats.reporting.lastDay.printable_pdf.app[app]).to.be(count);
-      expect(stats.reporting.last7Days.printable_pdf.app[app]).to.be(count);
+      expect(stats.reporting.last_day.printable_pdf.app[app]).to.be(count);
+      expect(stats.reporting.last_7_days.printable_pdf.app[app]).to.be(count);
     },
 
     expectAllTimePdfAppStats(stats, app, count) {
@@ -107,8 +107,8 @@ export function ReportingAPIProvider({ getService }) {
     },
 
     expectRecentPdfLayoutStats(stats, layout, count) {
-      expect(stats.reporting.lastDay.printable_pdf.layout[layout]).to.be(count);
-      expect(stats.reporting.last7Days.printable_pdf.layout[layout]).to.be(count);
+      expect(stats.reporting.last_day.printable_pdf.layout[layout]).to.be(count);
+      expect(stats.reporting.last_7_days.printable_pdf.layout[layout]).to.be(count);
     },
 
     expectAllTimePdfLayoutStats(stats, layout, count) {
@@ -116,8 +116,8 @@ export function ReportingAPIProvider({ getService }) {
     },
 
     expectRecentJobTypeTotalStats(stats, jobType, count) {
-      expect(stats.reporting.lastDay[jobType].total).to.be(count);
-      expect(stats.reporting.last7Days[jobType].total).to.be(count);
+      expect(stats.reporting.last_day[jobType].total).to.be(count);
+      expect(stats.reporting.last_7_days[jobType].total).to.be(count);
     },
 
     expectAllTimeJobTypeTotalStats(stats, jobType, count) {

From cb46d50e234ce671f8b6c6be6e9344867a212836 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Tue, 17 Jul 2018 12:49:20 -0700
Subject: [PATCH 37/42] more comment

---
 x-pack/test/reporting/api/usage.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/x-pack/test/reporting/api/usage.js b/x-pack/test/reporting/api/usage.js
index 3769d10796c70..8bfed1b2d5594 100644
--- a/x-pack/test/reporting/api/usage.js
+++ b/x-pack/test/reporting/api/usage.js
@@ -124,8 +124,9 @@ export default function ({ getService }) {
       });
     });
 
-    /* have to skip this test because the usage stats returned by the legacy
-     * endpoint aren't snake_cased in the legacy usage api
+    /* Have to skip this test because the usage stats returned by the legacy
+     * endpoint aren't snake_cased in the legacy usage api. This will be
+     * completely removed in the next PR, when the legacy endpoint is removed
      */
     describe.skip('deprecated API', () => {
       it('shows correct stats', async () => {

From 73e1d347da463d2b6c92a8e09f523536714e8e01 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Tue, 17 Jul 2018 16:10:11 -0700
Subject: [PATCH 38/42] stop putting a field in that used to be omitted

---
 .../kibana_monitoring/collectors/ops_buffer/ops_buffer.js     | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js
index 524b3fd434c05..bb7683899200c 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js
@@ -14,8 +14,6 @@ import { CloudDetector } from '../../../cloud';
  * @return {Object} the revealed `push` and `flush` modules
  */
 export function opsBuffer(server) {
-  let host = null;
-
   // determine the cloud service in the background
   const cloudDetector = new CloudDetector();
   cloudDetector.detectCloudService();
@@ -24,14 +22,12 @@ export function opsBuffer(server) {
 
   return {
     push(event) {
-      host = event.host;
       eventRoller.addEvent(event);
       server.log(['debug', LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG], 'Received Kibana Ops event data');
     },
 
     flush() {
       return {
-        host,
         cloud: cloudDetector.getCloudDetails(),
         ...eventRoller.flush()
       };

From 51444d7f199e5d2c5b841ba49f0335f9549413c8 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Tue, 17 Jul 2018 18:08:00 -0700
Subject: [PATCH 39/42] fix the internal type to ID the usage data for bulk
 uploader

---
 x-pack/plugins/monitoring/common/constants.js              | 2 +-
 .../bulk_uploader.combine_stats_legacy.test.js             | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/x-pack/plugins/monitoring/common/constants.js b/x-pack/plugins/monitoring/common/constants.js
index ed33d85dd41fd..4a2a11da9e9de 100644
--- a/x-pack/plugins/monitoring/common/constants.js
+++ b/x-pack/plugins/monitoring/common/constants.js
@@ -32,7 +32,7 @@ export const KIBANA_SETTINGS_TYPE = 'kibana_settings';
  * The type name used within the Monitoring index to publish Kibana usage stats.
  * @type {string}
  */
-export const KIBANA_USAGE_TYPE = 'kibana';
+export const KIBANA_USAGE_TYPE = 'kibana_usage';
 
 /*
  * Key for the localStorage service
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js
index e02d5d45f37fb..3526227939f66 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js
@@ -4,13 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { KIBANA_STATS_TYPE, KIBANA_USAGE_TYPE, KIBANA_SETTINGS_TYPE } from '../../common/constants';
 import { KIBANA_REPORTING_TYPE } from '../../../reporting/common/constants';
 import { BulkUploader } from './bulk_uploader';
 
 const getInitial = () => {
   return [
     [
-      { 'index': { '_type': 'kibana_stats' } },
+      { 'index': { '_type': KIBANA_STATS_TYPE } },
       {
         'host': 'tsullivan.local',
         'concurrent_connections': 0,
@@ -37,7 +38,7 @@ const getInitial = () => {
       }
     ],
     [
-      { 'index': { '_type': 'kibana' } },
+      { 'index': { '_type': KIBANA_USAGE_TYPE } },
       {
         'dashboard': { 'total': 0 },
         'visualization': { 'total': 0 },
@@ -63,7 +64,7 @@ const getInitial = () => {
       }
     ],
     [
-      { 'index': { '_type': 'kibana_settings' } },
+      { 'index': { '_type': KIBANA_SETTINGS_TYPE } },
       { 'xpack': { 'defaultAdminEmail': 'tim@elastic.co' } }
     ]
   ];

From 7102f04e2de6209004c6500aace104062b99b815 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Tue, 17 Jul 2018 22:59:32 -0700
Subject: [PATCH 40/42] correct the kibana usage type value, which is shown
 as-is in the API

---
 x-pack/plugins/monitoring/common/constants.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/monitoring/common/constants.js b/x-pack/plugins/monitoring/common/constants.js
index 4a2a11da9e9de..18f0f3f1fe720 100644
--- a/x-pack/plugins/monitoring/common/constants.js
+++ b/x-pack/plugins/monitoring/common/constants.js
@@ -30,9 +30,10 @@ export const KIBANA_STATS_TYPE = 'kibana_stats';
 export const KIBANA_SETTINGS_TYPE = 'kibana_settings';
 /**
  * The type name used within the Monitoring index to publish Kibana usage stats.
+ * NOTE: this string shows as-is in the stats API as a field name for the kibana usage stats
  * @type {string}
  */
-export const KIBANA_USAGE_TYPE = 'kibana_usage';
+export const KIBANA_USAGE_TYPE = 'kibana';
 
 /*
  * Key for the localStorage service

From 79111eb0aacfb2912fa33bb8c1a518a4aa0bdcd1 Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Wed, 18 Jul 2018 11:24:41 -0700
Subject: [PATCH 41/42] more fixes for the constants identifying collector
 types + test against duplicates

---
 src/server/status/constants.js                 |  2 +-
 x-pack/plugins/monitoring/common/constants.js  |  2 +-
 .../bulk_uploader.combine_stats_legacy.test.js | 16 ++++++++++++++--
 .../server/kibana_monitoring/bulk_uploader.js  | 18 +++++++++++++++---
 .../collectors/get_ops_stats_collector.js      |  4 ++--
 5 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/src/server/status/constants.js b/src/server/status/constants.js
index 1bd84dd1e4b03..d16e46008acc8 100644
--- a/src/server/status/constants.js
+++ b/src/server/status/constants.js
@@ -17,4 +17,4 @@
  * under the License.
  */
 
-export const KIBANA_STATS_TYPE = 'kibana';
+export const KIBANA_STATS_TYPE = 'kibana_stats'; // kibana stats per 5s intervals
diff --git a/x-pack/plugins/monitoring/common/constants.js b/x-pack/plugins/monitoring/common/constants.js
index 18f0f3f1fe720..30b700c48ae0e 100644
--- a/x-pack/plugins/monitoring/common/constants.js
+++ b/x-pack/plugins/monitoring/common/constants.js
@@ -22,7 +22,7 @@ export const MONITORING_SYSTEM_API_VERSION = '6';
  * The type name used within the Monitoring index to publish Kibana ops stats.
  * @type {string}
  */
-export const KIBANA_STATS_TYPE = 'kibana_stats';
+export const KIBANA_STATS_TYPE_MONITORING = 'kibana_stats_monitoring'; // similar to KIBANA_STATS_TYPE but rolled up into 10s stats from 5s intervals through ops_buffer
 /**
  * The type name used within the Monitoring index to publish Kibana stats.
  * @type {string}
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js
index 3526227939f66..9b71d9417d521 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js
@@ -4,14 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { KIBANA_STATS_TYPE, KIBANA_USAGE_TYPE, KIBANA_SETTINGS_TYPE } from '../../common/constants';
+import { KIBANA_STATS_TYPE_MONITORING, KIBANA_USAGE_TYPE, KIBANA_SETTINGS_TYPE } from '../../common/constants';
 import { KIBANA_REPORTING_TYPE } from '../../../reporting/common/constants';
 import { BulkUploader } from './bulk_uploader';
 
 const getInitial = () => {
   return [
     [
-      { 'index': { '_type': KIBANA_STATS_TYPE } },
+      { 'index': { '_type': KIBANA_STATS_TYPE_MONITORING } },
       {
         'host': 'tsullivan.local',
         'concurrent_connections': 0,
@@ -174,4 +174,16 @@ describe('Collector Types Combiner', () => {
       expect(result).toEqual(trimmedExpectedResult);
     });
   });
+
+  it('throws an error if duplicate types are registered', () => {
+    const combineWithDuplicate = () => {
+      const initial = getInitial();
+      const withDuplicate = [ initial[0] ].concat(initial);
+      return BulkUploader.combineStatsLegacy(withDuplicate);
+    };
+    expect(combineWithDuplicate).toThrow(
+      'Duplicate collector type identifiers found in payload! ' +
+      'kibana_stats_monitoring,kibana_stats_monitoring,kibana,reporting,kibana_settings'
+    );
+  });
 });
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
index ae11c5c36eb02..2eabe4fc9d20b 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
@@ -4,12 +4,12 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { get, set, isEmpty, flatten } from 'lodash';
+import { get, set, isEmpty, flatten, uniq } from 'lodash';
 import { callClusterFactory } from '../../../xpack_main';
 import {
   LOGGING_TAG,
   KIBANA_MONITORING_LOGGING_TAG,
-  KIBANA_STATS_TYPE,
+  KIBANA_STATS_TYPE_MONITORING,
   KIBANA_SETTINGS_TYPE,
   KIBANA_USAGE_TYPE,
 } from '../../common/constants';
@@ -131,18 +131,30 @@ export class BulkUploader {
     }
   }
 
+  static checkPayloadTypesUnique(payload) {
+    const ids = payload.map(item => item[0].index._type);
+    const uniques = uniq(ids);
+    if (ids.length !== uniques.length) {
+      throw new Error('Duplicate collector type identifiers found in payload! ' + ids.join(','));
+    }
+  }
+
   static combineStatsLegacy(payload) {
+    BulkUploader.checkPayloadTypesUnique(payload);
+
     // default the item to [] to allow destructuring
     const findItem = type => payload.find(item => get(item, '[0].index._type') === type) || [];
 
     // kibana usage and stats
     let statsResult;
-    const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE);
+    const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE_MONITORING);
     const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE);
 
     if (statsHeader && statsPayload) {
+      statsHeader.index._type = 'kibana_stats';
       const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE);
       const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null;
+
       const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null;
       statsResult = [ statsHeader, statsPayload ];
       if (kibanaUsage) {
diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
index 616d8afc3441f..1c79de29dea40 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { KIBANA_STATS_TYPE } from '../../../common/constants';
+import { KIBANA_STATS_TYPE_MONITORING } from '../../../common/constants';
 import { opsBuffer } from './ops_buffer';
 import { getKibanaInfoForStats } from '../lib';
 
@@ -45,7 +45,7 @@ export function getOpsStatsCollector(server, kbnServer) {
 
   const { collectorSet } = server.usage;
   return collectorSet.makeStatsCollector({
-    type: KIBANA_STATS_TYPE,
+    type: KIBANA_STATS_TYPE_MONITORING,
     init: start,
     fetch: () => {
       return {

From 528cc0d347d0d433aa46f9a01776d5888f9b5b0b Mon Sep 17 00:00:00 2001
From: Timothy Sullivan <tsullivan@elastic.co>
Date: Wed, 18 Jul 2018 11:43:31 -0700
Subject: [PATCH 42/42] add a comment on a hack, and a whitespace fix

---
 .../monitoring/server/kibana_monitoring/bulk_uploader.js       | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
index 2eabe4fc9d20b..c95baa056c8d6 100644
--- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
+++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js
@@ -151,10 +151,9 @@ export class BulkUploader {
     const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE);
 
     if (statsHeader && statsPayload) {
-      statsHeader.index._type = 'kibana_stats';
+      statsHeader.index._type = 'kibana_stats'; // HACK to convert kibana_stats_monitoring to just kibana_stats for bwc
       const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE);
       const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null;
-
       const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null;
       statsResult = [ statsHeader, statsPayload ];
       if (kibanaUsage) {