Skip to content

Commit

Permalink
[7.x] initial telemetry setup (elastic#69330) (elastic#71578)
Browse files Browse the repository at this point in the history
  • Loading branch information
michaelolo24 authored Jul 14, 2020
1 parent 8ca6757 commit 2d3b157
Show file tree
Hide file tree
Showing 12 changed files with 546 additions and 25 deletions.
1 change: 1 addition & 0 deletions x-pack/plugins/security_solution/server/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
initSavedObjects(core.savedObjects);
initUiSettings(core.uiSettings);
initUsageCollectors({
core,
kibanaIndex: globalConfig.kibana.index,
ml: plugins.ml,
usageCollection: plugins.usageCollection,
Expand Down
45 changes: 39 additions & 6 deletions x-pack/plugins/security_solution/server/usage/collector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,32 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { LegacyAPICaller } from '../../../../../src/core/server';
import { LegacyAPICaller, CoreSetup } from '../../../../../src/core/server';
import { CollectorDependencies } from './types';
import { DetectionsUsage, fetchDetectionsUsage } from './detections';
import { EndpointUsage, getEndpointTelemetryFromFleet } from './endpoints';

export type RegisterCollector = (deps: CollectorDependencies) => void;
export interface UsageData {
detections: DetectionsUsage;
endpoints: EndpointUsage;
}

export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCollection }) => {
export async function getInternalSavedObjectsClient(core: CoreSetup) {
return core.getStartServices().then(async ([coreStart]) => {
return coreStart.savedObjects.createInternalRepository();
});
}

export const registerCollector: RegisterCollector = ({
core,
kibanaIndex,
ml,
usageCollection,
}) => {
if (!usageCollection) {
return;
}

const collector = usageCollection.makeUsageCollector<UsageData>({
type: 'security_solution',
schema: {
Expand All @@ -43,11 +55,32 @@ export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCol
},
},
},
endpoints: {
total_installed: { type: 'long' },
active_within_last_24_hours: { type: 'long' },
os: {
full_name: { type: 'keyword' },
platform: { type: 'keyword' },
version: { type: 'keyword' },
count: { type: 'long' },
},
policies: {
malware: {
success: { type: 'long' },
warning: { type: 'long' },
failure: { type: 'long' },
},
},
},
},
isReady: () => kibanaIndex.length > 0,
fetch: async (callCluster: LegacyAPICaller): Promise<UsageData> => ({
detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml),
}),
fetch: async (callCluster: LegacyAPICaller): Promise<UsageData> => {
const savedObjectsClient = await getInternalSavedObjectsClient(core);
return {
detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml),
endpoints: await getEndpointTelemetryFromFleet(savedObjectsClient),
};
},
});

usageCollection.registerCollector(collector);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { INTERNAL_IMMUTABLE_KEY } from '../../common/constants';
import { INTERNAL_IMMUTABLE_KEY } from '../../../common/constants';

export const getMockJobSummaryResponse = () => [
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { LegacyAPICaller } from '../../../../../src/core/server';
import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
import { jobServiceProvider } from '../../../ml/server/models/job_service';
import { DataRecognizer } from '../../../ml/server/models/data_recognizer';
import { mlServicesMock } from '../lib/machine_learning/mocks';
import { LegacyAPICaller } from '../../../../../../src/core/server';
import { elasticsearchServiceMock } from '../../../../../../src/core/server/mocks';
import { jobServiceProvider } from '../../../../ml/server/models/job_service';
import { DataRecognizer } from '../../../../ml/server/models/data_recognizer';
import { mlServicesMock } from '../../lib/machine_learning/mocks';
import {
getMockJobSummaryResponse,
getMockListModulesResponse,
getMockRulesResponse,
} from './detections.mocks';
import { fetchDetectionsUsage } from './detections';
import { fetchDetectionsUsage } from './index';

jest.mock('../../../ml/server/models/job_service');
jest.mock('../../../ml/server/models/data_recognizer');
jest.mock('../../../../ml/server/models/job_service');
jest.mock('../../../../ml/server/models/data_recognizer');

describe('Detections Usage', () => {
describe('fetchDetectionsUsage()', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@

import { SearchParams } from 'elasticsearch';

import { LegacyAPICaller, SavedObjectsClient } from '../../../../../src/core/server';
import { LegacyAPICaller, SavedObjectsClient } from '../../../../../../src/core/server';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { jobServiceProvider } from '../../../ml/server/models/job_service';
import { jobServiceProvider } from '../../../../ml/server/models/job_service';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { DataRecognizer } from '../../../ml/server/models/data_recognizer';
import { MlPluginSetup } from '../../../ml/server';
import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../common/constants';
import { DetectionRulesUsage, MlJobsUsage } from './detections';
import { isJobStarted } from '../../common/machine_learning/helpers';
import { DataRecognizer } from '../../../../ml/server/models/data_recognizer';
import { MlPluginSetup } from '../../../../ml/server';
import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../../common/constants';
import { DetectionRulesUsage, MlJobsUsage } from './index';
import { isJobStarted } from '../../../common/machine_learning/helpers';

interface DetectionsMetric {
isElastic: boolean;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { LegacyAPICaller } from '../../../../../src/core/server';
import { LegacyAPICaller } from '../../../../../../src/core/server';
import { getMlJobsUsage, getRulesUsage } from './detections_helpers';
import { MlPluginSetup } from '../../../ml/server';
import { MlPluginSetup } from '../../../../ml/server';

interface FeatureUsage {
enabled: number;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObjectsFindResponse } from 'src/core/server';
import { AgentEventSOAttributes } from './../../../../ingest_manager/common/types/models/agent';
import {
AGENT_SAVED_OBJECT_TYPE,
AGENT_EVENT_SAVED_OBJECT_TYPE,
} from '../../../../ingest_manager/common/constants/agent';
import { Agent } from '../../../../ingest_manager/common';
import { FLEET_ENDPOINT_PACKAGE_CONSTANT } from './fleet_saved_objects';

const testAgentId = 'testAgentId';
const testConfigId = 'testConfigId';

/** Mock OS Platform for endpoint telemetry */
export const MockOSPlatform = 'somePlatform';
/** Mock OS Name for endpoint telemetry */
export const MockOSName = 'somePlatformName';
/** Mock OS Version for endpoint telemetry */
export const MockOSVersion = '1';
/** Mock OS Full Name for endpoint telemetry */
export const MockOSFullName = 'somePlatformFullName';

/**
*
* @param lastCheckIn - the last time the agent checked in. Defaults to current ISO time.
* @description We request the install and OS related telemetry information from the 'fleet-agents' saved objects in ingest_manager. This mocks that response
*/
export const mockFleetObjectsResponse = (
lastCheckIn = new Date().toISOString()
): SavedObjectsFindResponse<Agent> => ({
page: 1,
per_page: 20,
total: 1,
saved_objects: [
{
type: AGENT_SAVED_OBJECT_TYPE,
id: testAgentId,
attributes: {
active: true,
id: testAgentId,
config_id: 'randoConfigId',
type: 'PERMANENT',
user_provided_metadata: {},
enrolled_at: lastCheckIn,
current_error_events: [],
local_metadata: {
elastic: {
agent: {
id: testAgentId,
},
},
host: {
hostname: 'testDesktop',
name: 'testDesktop',
id: 'randoHostId',
},
os: {
platform: MockOSPlatform,
version: MockOSVersion,
name: MockOSName,
full: MockOSFullName,
},
},
packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'],
last_checkin: lastCheckIn,
},
references: [],
updated_at: lastCheckIn,
version: 'WzI4MSwxXQ==',
score: 0,
},
],
});

/**
*
* @param running - allows us to set whether the mocked endpoint is in an active or disabled/failed state
* @param updatedDate - the last time the endpoint was updated. Defaults to current ISO time.
* @description We request the events triggered by the agent and get the most recent endpoint event to confirm it is still running. This allows us to mock both scenarios
*/
export const mockFleetEventsObjectsResponse = (
running?: boolean,
updatedDate = new Date().toISOString()
): SavedObjectsFindResponse<AgentEventSOAttributes> => {
return {
page: 1,
per_page: 20,
total: 2,
saved_objects: [
{
type: AGENT_EVENT_SAVED_OBJECT_TYPE,
id: 'id1',
attributes: {
agent_id: testAgentId,
type: running ? 'STATE' : 'ERROR',
timestamp: updatedDate,
subtype: running ? 'RUNNING' : 'FAILED',
message: `Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to ${
running ? 'RUNNING' : 'FAILED'
}: `,
config_id: testConfigId,
},
references: [],
updated_at: updatedDate,
version: 'WzExOCwxXQ==',
score: 0,
},
{
type: AGENT_EVENT_SAVED_OBJECT_TYPE,
id: 'id2',
attributes: {
agent_id: testAgentId,
type: 'STATE',
timestamp: updatedDate,
subtype: 'STARTING',
message:
'Application: endpoint-security--8.0.0[d8f7f6e8-9375-483c-b456-b479f1d7a4f2]: State changed to STARTING: Starting',
config_id: testConfigId,
},
references: [],
updated_at: updatedDate,
version: 'WzExNywxXQ==',
score: 0,
},
],
};
};
Loading

0 comments on commit 2d3b157

Please sign in to comment.