From bff79fc2fb35e951f530fb39bc13264bc38d1fa8 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Wed, 12 May 2021 14:34:49 -0400 Subject: [PATCH] Indexing of generated data changed to index fake fleet server agents --- .../common/endpoint/index_data.ts | 249 ++++-------------- 1 file changed, 48 insertions(+), 201 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index fd26a2d95c9b4..8dfff9d00a034 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -5,31 +5,28 @@ * 2.0. */ -import { Client } from '@elastic/elasticsearch'; +import { Client, estypes } from '@elastic/elasticsearch'; import seedrandom from 'seedrandom'; // eslint-disable-next-line import/no-extraneous-dependencies import { KbnClient } from '@kbn/test'; import { AxiosResponse } from 'axios'; -import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; +import { EndpointDocGenerator, Event, TreeOptions } from './generate_data'; import { firstNonNullValue } from './models/ecs_safety_helpers'; import { + AGENT_POLICY_API_ROUTES, CreateAgentPolicyRequest, CreateAgentPolicyResponse, CreatePackagePolicyRequest, CreatePackagePolicyResponse, - GetPackagesResponse, - AGENT_API_ROUTES, - AGENT_POLICY_API_ROUTES, EPM_API_ROUTES, + FleetServerAgent, + GetPackagesResponse, PACKAGE_POLICY_API_ROUTES, - ENROLLMENT_API_KEY_ROUTES, - GetEnrollmentAPIKeysResponse, - GetOneEnrollmentAPIKeyResponse, - Agent, } from '../../../fleet/common'; import { policyFactory as policyConfigFactory } from './models/policy_config'; import { HostMetadata } from './types'; import { KbnClientWithApiKeySupport } from '../../scripts/endpoint/kbn_client_with_api_key_support'; +import { FleetAgentGenerator } from './data_generators/fleet_agent_generator'; export async function indexHostsAndAlerts( client: Client, @@ -107,9 +104,11 @@ async function indexHostDocs({ }) { const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents const timestamp = new Date().getTime(); + const fleetAgentGenerator = new FleetAgentGenerator(); + const kibanaVersion = await fetchKibanaVersion(kbnClient); let hostMetadata: HostMetadata; let wasAgentEnrolled = false; - let enrolledAgent: undefined | Agent; + let enrolledAgent: undefined | estypes.Hit; for (let j = 0; j < numDocs; j++) { generator.updateHostData(); @@ -136,10 +135,13 @@ async function indexHostDocs({ // If we did not yet enroll an agent for this Host, do it now that we have good policy id if (!wasAgentEnrolled) { wasAgentEnrolled = true; - enrolledAgent = await fleetEnrollAgentForHost( - kbnClient, + + enrolledAgent = await indexFleetAgentForHost( + client, + fleetAgentGenerator, hostMetadata!, - realPolicies[appliedPolicyId].policy_id + realPolicies[appliedPolicyId].policy_id, + kibanaVersion ); } // Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id @@ -149,7 +151,7 @@ async function indexHostDocs({ ...hostMetadata.elastic, agent: { ...hostMetadata.elastic.agent, - id: enrolledAgent?.id ?? hostMetadata.elastic.agent.id, + id: enrolledAgent?._id ?? hostMetadata.elastic.agent.id, }, }, Endpoint: { @@ -295,208 +297,53 @@ const getEndpointPackageInfo = async ( return endpointPackage; }; -const fleetEnrollAgentForHost = async ( - kbnClient: KbnClientWithApiKeySupport, - endpointHost: HostMetadata, - agentPolicyId: string -): Promise => { - // Get Enrollement key for host's applied policy - const enrollmentApiKey = await kbnClient - .request({ - path: ENROLLMENT_API_KEY_ROUTES.LIST_PATTERN, - method: 'GET', - query: { - kuery: `policy_id:"${agentPolicyId}"`, - }, - }) - .then((apiKeysResponse) => { - const apiKey = apiKeysResponse.data.list[0]; - - if (!apiKey) { - return Promise.reject( - new Error(`no API enrollment key found for agent policy id ${agentPolicyId}`) - ); - } - - return kbnClient - .request({ - path: ENROLLMENT_API_KEY_ROUTES.INFO_PATTERN.replace('{keyId}', apiKey.id), - method: 'GET', - }) - .catch((error) => { - // eslint-disable-next-line no-console - console.log('unable to retrieve enrollment api key for policy'); - return Promise.reject(error); - }); - }) - .then((apiKeyDetailsResponse) => { - return apiKeyDetailsResponse.data.item.api_key; - }) - .catch((error) => { - // eslint-disable-next-line no-console - console.error(error); - return ''; - }); +const fetchKibanaVersion = async (kbnClient: KbnClientWithApiKeySupport) => { + const version = ((await kbnClient.request({ + path: '/api/status', + method: 'GET', + })) as AxiosResponse).data.version.number; - if (enrollmentApiKey.length === 0) { - return; + if (!version) { + // eslint-disable-next-line no-console + console.log('failed to retrieve kibana version'); + return '8.0.0'; } - const fetchKibanaVersion = async () => { - const version = ((await kbnClient.request({ - path: '/api/status', - method: 'GET', - })) as AxiosResponse).data.version.number; - if (!version) { - // eslint-disable-next-line no-console - console.log('failed to retrieve kibana version'); - } - return version; - }; + return version; +}; - // Enroll an agent for the Host - const body = { - type: 'PERMANENT', - metadata: { - local: { +const indexFleetAgentForHost = async ( + esClient: Client, + fleetAgentGenerator: FleetAgentGenerator, + endpointHost: HostMetadata, + agentPolicyId: string, + kibanaVersion: string = '8.0.0' +): Promise> => { + const agentDoc = fleetAgentGenerator.generateEsHit({ + _source: { + local_metadata: { elastic: { agent: { - version: await fetchKibanaVersion(), + version: kibanaVersion, }, }, host: { ...endpointHost.host, }, os: { - family: 'windows', - kernel: '10.0.19041.388 (WinBuild.160101.0800)', - platform: 'windows', - version: '10.0', - name: 'Windows 10 Pro', - full: 'Windows 10 Pro(10.0)', + ...endpointHost.host.os, }, }, - user_provided: { - dev_agent_version: '0.0.1', - region: 'us-east', - }, + policy_id: agentPolicyId, }, - }; - - try { - // First enroll the agent - const res = await kbnClient.requestWithApiKey(AGENT_API_ROUTES.ENROLL_PATTERN, { - method: 'POST', - body: JSON.stringify(body), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollmentApiKey}`, - 'Content-Type': 'application/json', - }, - }); - - if (res) { - const enrollObj = await res.json(); - if (!res.ok) { - // eslint-disable-next-line no-console - console.error('unable to enroll agent', enrollObj); - return; - } - // ------------------------------------------------ - // now check the agent in so that it can complete enrollment - const checkinBody = { - events: [ - { - type: 'STATE', - subtype: 'RUNNING', - message: 'state changed from STOPPED to RUNNING', - timestamp: new Date().toISOString(), - payload: { - random: 'data', - state: 'RUNNING', - previous_state: 'STOPPED', - }, - agent_id: enrollObj.item.id, - }, - ], - }; - const checkinRes = await kbnClient - .requestWithApiKey( - AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id), - { - method: 'POST', - body: JSON.stringify(checkinBody), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollObj.item.access_api_key}`, - 'Content-Type': 'application/json', - }, - } - ) - .catch((error) => { - return Promise.reject(error); - }); - - // Agent unenrolling? - if (checkinRes.status === 403) { - return; - } - - const checkinObj = await checkinRes.json(); - if (!checkinRes.ok) { - // eslint-disable-next-line no-console - console.error( - `failed to checkin agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` - ); - return enrollObj.item; - } - - // ------------------------------------------------ - // If we have an action to ack(), then do it now - if (checkinObj.actions.length) { - const ackActionBody = { - // @ts-ignore - events: checkinObj.actions.map((action) => { - return { - action_id: action.id, - type: 'ACTION_RESULT', - subtype: 'CONFIG', - timestamp: new Date().toISOString(), - agent_id: action.agent_id, - policy_id: agentPolicyId, - message: `endpoint generator: Endpoint Started`, - }; - }), - }; - const ackActionResp = await kbnClient.requestWithApiKey( - AGENT_API_ROUTES.ACKS_PATTERN.replace('{agentId}', enrollObj.item.id), - { - method: 'POST', - body: JSON.stringify(ackActionBody), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollObj.item.access_api_key}`, - 'Content-Type': 'application/json', - }, - } - ); + }); - const ackActionObj = await ackActionResp.json(); - if (!ackActionResp.ok) { - // eslint-disable-next-line no-console - console.error( - `failed to ACK Actions provided to agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` - ); - // eslint-disable-next-line no-console - console.error(JSON.stringify(ackActionObj, null, 2)); - return enrollObj.item; - } - } + await esClient.index({ + index: agentDoc._index, + id: agentDoc._id, + body: agentDoc._source!, + op_type: 'create', + }); - return enrollObj.item; - } - } catch (error) { - // eslint-disable-next-line no-console - console.error(error); - } + return agentDoc; };