[Telemetry] Remove from and to timestamps from usage stats APIs

src/plugins/telemetry/public/services/telemetry_service.test.ts

@@ -18,42 +18,30 @@
  */
 
 /* eslint-disable dot-notation */
-import { mockTelemetryService } from '../mocks';
-
-const mockSubtract = jest.fn().mockImplementation(() => {
-  return {
-    toISOString: jest.fn(),
-  };
-});
-
-const mockClone = jest.fn().mockImplementation(() => {
-  return {
-    clone: mockClone,
-    subtract: mockSubtract,
-    toISOString: jest.fn(),
-  };
-});
+const mockMomentValueOf = jest.fn();
 
 jest.mock('moment', () => {
   return jest.fn().mockImplementation(() => {
     return {
-      clone: mockClone,
-      subtract: mockSubtract,
-      toISOString: jest.fn(),
+      valueOf: mockMomentValueOf,
     };
   });
 });
 
+import { mockTelemetryService } from '../mocks';
+
 describe('TelemetryService', () => {
   describe('fetchTelemetry', () => {
     it('calls expected URL with 20 minutes - now', async () => {
+      const timestamp = Date.now();
+      mockMomentValueOf.mockReturnValueOnce(timestamp);
       const telemetryService = mockTelemetryService();
+
       await telemetryService.fetchTelemetry();
       expect(telemetryService['http'].post).toBeCalledWith('/api/telemetry/v2/clusters/_stats', {
-        body: JSON.stringify({ unencrypted: false, timeRange: {} }),
+        body: JSON.stringify({ unencrypted: false, timestamp }),
       });
-      expect(mockClone).toBeCalled();
-      expect(mockSubtract).toBeCalledWith(20, 'minutes');
+      expect(mockMomentValueOf).toBeCalled();
     });
   });
 

src/plugins/telemetry/public/services/telemetry_service.ts

@@ -121,17 +121,10 @@ export class TelemetryService {
   };
 
   public fetchTelemetry = async ({ unencrypted = false } = {}) => {
-    const now = moment();
     return this.http.post('/api/telemetry/v2/clusters/_stats', {
       body: JSON.stringify({
         unencrypted,
-        timeRange: {
-          min: now
-            .clone() // Need to clone it to avoid mutation (and max being the same value)
-            .subtract(20, 'minutes')
-            .toISOString(),
-          max: now.toISOString(),
-        },
+        timestamp: moment().valueOf(),
       }),
     });
   };

src/plugins/telemetry/server/fetcher.ts

@@ -213,8 +213,7 @@ export class FetcherTask {
   private async fetchTelemetry() {
     return await this.telemetryCollectionManager!.getStats({
       unencrypted: false,
-      start: moment().subtract(20, 'minutes').toISOString(),
-      end: moment().toISOString(),
+      timestamp: moment().valueOf(),
     });
   }
 

src/plugins/telemetry/server/routes/telemetry_opt_in.ts

@@ -86,8 +86,7 @@ export function registerTelemetryOptInRoutes({
       }
 
       const statsGetterConfig: StatsGetterConfig = {
-        start: moment().subtract(20, 'minutes').toISOString(),
-        end: moment().toISOString(),
+        timestamp: moment().valueOf(),
         unencrypted: false,
       };
 

src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts

@@ -72,8 +72,7 @@ export function registerTelemetryOptInStatsRoutes(
         const unencrypted = req.body.unencrypted;
 
         const statsGetterConfig: StatsGetterConfig = {
-          start: moment().subtract(20, 'minutes').toISOString(),
-          end: moment().toISOString(),
+          timestamp: moment().valueOf(),
           unencrypted,
           request: req,
         };

src/plugins/telemetry/server/routes/telemetry_usage_stats.ts

@@ -32,8 +32,6 @@ const validate: TypeOptions<string | number>['validate'] = (value) => {
   }
 };
 
-const dateSchema = schema.oneOf([schema.string({ validate }), schema.number({ validate })]);
-
 export function registerTelemetryUsageStatsRoutes(
   router: IRouter,
   telemetryCollectionManager: TelemetryCollectionManagerPluginSetup,
@@ -45,25 +43,20 @@ export function registerTelemetryUsageStatsRoutes(
       validate: {
         body: schema.object({
           unencrypted: schema.boolean({ defaultValue: false }),
-          timeRange: schema.object({
-            min: dateSchema,
-            max: dateSchema,
-          }),
+          timestamp: schema.oneOf([schema.string({ validate }), schema.number({ validate })]),
         }),
       },
     },
     async (context, req, res) => {
-      const start = moment(req.body.timeRange.min).toISOString();
-      const end = moment(req.body.timeRange.max).toISOString();
-      const unencrypted = req.body.unencrypted;
+      const { unencrypted, timestamp } = req.body;
 
       try {
         const statsConfig: StatsGetterConfig = {
-          unencrypted,
-          start,
-          end,
+          timestamp: moment(timestamp).valueOf(),
           request: req,
+          unencrypted,
         };
+
         const stats = await telemetryCollectionManager.getStats(statsConfig);
         return res.ok({ body: stats });
       } catch (err) {

src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts

@@ -87,8 +87,7 @@ function mockStatsCollectionConfig(clusterInfo: any, clusterStats: any, kibana:
     ...createCollectorFetchContextMock(),
     esClient: mockGetLocalStats(clusterInfo, clusterStats),
     usageCollection: mockUsageCollection(kibana),
-    start: '',
-    end: '',
+    timestamp: Date.now(),
   };
 }
 

src/plugins/telemetry_collection_manager/server/plugin.ts

@@ -144,7 +144,7 @@ export class TelemetryCollectionManagerPlugin
     collectionSoService: SavedObjectsServiceStart,
     usageCollection: UsageCollectionSetup
   ): StatsCollectionConfig {
-    const { start, end, request } = config;
+    const { timestamp, request } = config;
 
     const callCluster = config.unencrypted
       ? collection.esCluster.asScoped(request).callAsCurrentUser
@@ -157,7 +157,7 @@ export class TelemetryCollectionManagerPlugin
     const soClient = config.unencrypted
       ? collectionSoService.getScopedClient(config.request)
       : collectionSoService.createInternalRepository();
-    return { callCluster, start, end, usageCollection, esClient, soClient };
+    return { callCluster, timestamp, usageCollection, esClient, soClient };
   }
 
   private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) {

src/plugins/telemetry_collection_manager/server/types.ts

@@ -56,8 +56,7 @@ export interface TelemetryOptInStats {
 
 export interface BaseStatsGetterConfig {
   unencrypted: boolean;
-  start: string;
-  end: string;
+  timestamp: number;
   request?: KibanaRequest;
 }
 
@@ -77,8 +76,7 @@ export interface ClusterDetails {
 export interface StatsCollectionConfig {
   usageCollection: UsageCollectionSetup;
   callCluster: LegacyAPICaller;
-  start: string | number;
-  end: string | number;
+  timestamp: number;
   esClient: ElasticsearchClient;
   soClient: SavedObjectsClientContract | ISavedObjectsRepository;
 }

test/api_integration/apis/telemetry/telemetry_local.js

@@ -53,15 +53,12 @@ export default function ({ getService }) {
     });
 
     it('should pull local stats and validate data types', async () => {
-      const timeRange = {
-        min: '2018-07-23T22:07:00Z',
-        max: '2018-07-23T22:13:00Z',
-      };
+      const timestamp = '2018-07-23T22:13:00Z';
 
       const { body } = await supertest
         .post('/api/telemetry/v2/clusters/_stats')
         .set('kbn-xsrf', 'xxx')
-        .send({ timeRange, unencrypted: true })
+        .send({ timestamp, unencrypted: true })
         .expect(200);
 
       expect(body.length).to.be(1);
@@ -98,15 +95,12 @@ export default function ({ getService }) {
     });
 
     it('should pull local stats and validate fields', async () => {
-      const timeRange = {
-        min: '2018-07-23T22:07:00Z',
-        max: '2018-07-23T22:13:00Z',
-      };
+      const timestamp = '2018-07-23T22:13:00Z';
 
       const { body } = await supertest
         .post('/api/telemetry/v2/clusters/_stats')
         .set('kbn-xsrf', 'xxx')
-        .send({ timeRange, unencrypted: true })
+        .send({ timestamp, unencrypted: true })
         .expect(200);
 
       const stats = body[0];
@@ -156,10 +150,7 @@ export default function ({ getService }) {
     });
 
     describe('application usage limits', () => {
-      const timeRange = {
-        min: '2018-07-23T22:07:00Z',
-        max: '2018-07-23T22:13:00Z',
-      };
+      const timestamp = '2018-07-23T22:13:00Z';
 
       function createSavedObject() {
         return supertest
@@ -191,7 +182,7 @@ export default function ({ getService }) {
           const { body } = await supertest
             .post('/api/telemetry/v2/clusters/_stats')
             .set('kbn-xsrf', 'xxx')
-            .send({ timeRange, unencrypted: true })
+            .send({ timestamp, unencrypted: true })
             .expect(200);
 
           expect(body.length).to.be(1);
@@ -242,7 +233,7 @@ export default function ({ getService }) {
           const { body } = await supertest
             .post('/api/telemetry/v2/clusters/_stats')
             .set('kbn-xsrf', 'xxx')
-            .send({ timeRange, unencrypted: true })
+            .send({ timestamp, unencrypted: true })
             .expect(200);
 
           expect(body.length).to.be(1);

x-pack/dev-tools/api_debug/apis/telemetry/index.js

@@ -7,18 +7,8 @@
 import moment from 'moment';
 
 export const name = 'telemetry';
-export const description = 'Get the clusters stats for the last 1 hour from the Kibana server';
+export const description = 'Get the clusters stats from the Kibana server';
 export const method = 'POST';
 export const path = '/api/telemetry/v2/clusters/_stats';
 
-// Get an object with start and end times for the last 1 hour, ISO format, in UTC
-function getTimeRange() {
-  const end = moment();
-  const start = moment(end).subtract(1, 'hour');
-  return {
-    min: moment.utc(start).format(),
-    max: moment.utc(end).format(),
-  };
-}
-
-export const body = { timeRange: getTimeRange(), unencrypted: true };
+export const body = { timeRange: moment().valueOf(), unencrypted: true };

x-pack/plugins/monitoring/common/constants.ts

@@ -219,6 +219,22 @@ export const REPORTING_SYSTEM_ID = 'reporting';
  */
 export const TELEMETRY_COLLECTION_INTERVAL = 86400000;
 
+/**
+ * The amount of time, in milliseconds, to fetch the cluster uuids from es.
+ *
+ * Currently 3 hours.
+ * @type {Number}
+ */
+export const CLUSTER_DETAILS_FETCH_INTERVAL = 10800000;
+
+/**
+ * The amount of time, in milliseconds, to fetch the usage data from es.
+ *
+ * Currently 20 minutes.
+ * @type {Number}
+ */
+export const USAGE_FETCH_INTERVAL = 1200000;
+
 /**
  * The prefix for all alert types used by monitoring
  */

x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts

@@ -12,8 +12,7 @@ import { ClustersHighLevelStats } from './get_high_level_stats';
 import { coreMock } from 'src/core/server/mocks';
 
 describe('get_all_stats', () => {
-  const start = 0;
-  const end = 1;
+  const timestamp = Date.now();
   const callCluster = sinon.stub();
   const esClient = sinon.stub();
   const soClient = sinon.stub();
@@ -181,8 +180,7 @@ describe('get_all_stats', () => {
             esClient: esClient as any,
             soClient: soClient as any,
             usageCollection: {} as any,
-            start,
-            end,
+            timestamp,
           },
           {
             logger: coreMock.createPluginInitializerContext().logger.get('test'),
@@ -208,8 +206,7 @@ describe('get_all_stats', () => {
             esClient: esClient as any,
             soClient: soClient as any,
             usageCollection: {} as any,
-            start,
-            end,
+            timestamp,
           },
           {
             logger: coreMock.createPluginInitializerContext().logger.get('test'),

x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts

@@ -8,15 +8,17 @@ import { set } from '@elastic/safer-lodash-set';
 import { get, merge } from 'lodash';
 
 import { StatsGetter } from 'src/plugins/telemetry_collection_manager/server';
-import { LOGSTASH_SYSTEM_ID, KIBANA_SYSTEM_ID, BEATS_SYSTEM_ID } from '../../common/constants';
+import moment from 'moment';
+import {
+  LOGSTASH_SYSTEM_ID,
+  KIBANA_SYSTEM_ID,
+  BEATS_SYSTEM_ID,
+  USAGE_FETCH_INTERVAL,
+} from '../../common/constants';
 import { getElasticsearchStats, ESClusterStats } from './get_es_stats';
 import { getKibanaStats, KibanaStats } from './get_kibana_stats';
-import { getBeatsStats } from './get_beats_stats';
-import { getHighLevelStats } from './get_high_level_stats';
-
-type PromiseReturnType<T extends (...args: any[]) => any> = ReturnType<T> extends Promise<infer R>
-  ? R
-  : T;
+import { getBeatsStats, BeatsStatsByClusterUuid } from './get_beats_stats';
+import { getHighLevelStats, ClustersHighLevelStats } from './get_high_level_stats';
 
 export interface CustomContext {
   maxBucketSize: number;
@@ -28,9 +30,12 @@ export interface CustomContext {
  */
 export const getAllStats: StatsGetter<CustomContext> = async (
   clustersDetails,
-  { callCluster, start, end, esClient, soClient },
+  { callCluster, timestamp },
   { maxBucketSize }
 ) => {
+  const start = moment(timestamp).subtract(USAGE_FETCH_INTERVAL, 'ms').toISOString();
+  const end = moment(timestamp).toISOString();
+
   const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid);
 
   const [esClusters, kibana, logstash, beats] = await Promise.all([
@@ -61,8 +66,8 @@ export function handleAllStats(
     beats,
   }: {
     kibana: KibanaStats;
-    logstash: PromiseReturnType<typeof getHighLevelStats>;
-    beats: PromiseReturnType<typeof getBeatsStats>;
+    logstash: ClustersHighLevelStats;
+    beats: BeatsStatsByClusterUuid;
   }
 ) {
   return clusters.map((cluster) => {

x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.test.ts

@@ -20,8 +20,8 @@ const getBaseOptions = () => ({
 describe('Get Beats Stats', () => {
   describe('fetchBeatsStats', () => {
     const clusterUuids = ['aCluster', 'bCluster', 'cCluster'];
-    const start = 100;
-    const end = 200;
+    const start = new Date().toISOString();
+    const end = new Date().toISOString();
     let callCluster = sinon.stub();
 
     beforeEach(() => {