From 07abacda7dccfdc14d4f28fd3d12a93e4be3c97c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 17 May 2018 09:18:48 -0700 Subject: [PATCH] [Beats Management] APIs: Verify beats (#19103) * WIP checkin * WIP checkin * Add API integration test * Converting to Jest test * Fixing API for default case + adding test for it * Fixing copy pasta typos * Fixing variable name * Using a single index * Implementing GET /api/beats/agents API * Creating POST /api/beats/agents/verify API * Refactoring: extracting out helper functions * Fleshing out remaining tests * Expanding TODO note so I won't forget :) * Fixing file name * Updating mapping * Moving TODO comment to right file * Rename determine* helper functions to find* --- .../plugins/beats/server/routes/api/index.js | 2 + .../routes/api/register_verify_beats_route.js | 143 ++++++++++++++++++ .../test/api_integration/apis/beats/index.js | 1 + .../apis/beats/verify_beats.js | 81 ++++++++++ .../es_archives/beats/list/data.json.gz | Bin 343 -> 371 bytes 5 files changed, 227 insertions(+) create mode 100644 x-pack/plugins/beats/server/routes/api/register_verify_beats_route.js create mode 100644 x-pack/test/api_integration/apis/beats/verify_beats.js diff --git a/x-pack/plugins/beats/server/routes/api/index.js b/x-pack/plugins/beats/server/routes/api/index.js index 76cedde5cdf3d..def322f0e94eb 100644 --- a/x-pack/plugins/beats/server/routes/api/index.js +++ b/x-pack/plugins/beats/server/routes/api/index.js @@ -7,9 +7,11 @@ import { registerCreateEnrollmentTokensRoute } from './register_create_enrollment_tokens_route'; import { registerEnrollBeatRoute } from './register_enroll_beat_route'; import { registerListBeatsRoute } from './register_list_beats_route'; +import { registerVerifyBeatsRoute } from './register_verify_beats_route'; export function registerApiRoutes(server) { registerCreateEnrollmentTokensRoute(server); registerEnrollBeatRoute(server); registerListBeatsRoute(server); + registerVerifyBeatsRoute(server); } diff --git a/x-pack/plugins/beats/server/routes/api/register_verify_beats_route.js b/x-pack/plugins/beats/server/routes/api/register_verify_beats_route.js new file mode 100644 index 0000000000000..11a4aff1204dc --- /dev/null +++ b/x-pack/plugins/beats/server/routes/api/register_verify_beats_route.js @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Joi from 'joi'; +import moment from 'moment'; +import { + get, + flatten +} from 'lodash'; +import { INDEX_NAMES } from '../../../common/constants'; +import { callWithRequestFactory } from '../../lib/client'; +import { wrapEsError } from '../../lib/error_wrappers'; + +async function getBeats(callWithRequest, beatIds) { + const ids = beatIds.map(beatId => `beat:${beatId}`); + const params = { + index: INDEX_NAMES.BEATS, + type: '_doc', + body: { ids }, + _sourceInclude: [ 'beat.id', 'beat.verified_on' ] + }; + + const response = await callWithRequest('mget', params); + return get(response, 'docs', []); +} + +async function verifyBeats(callWithRequest, beatIds) { + if (!Array.isArray(beatIds) || (beatIds.length === 0)) { + return []; + } + + const verifiedOn = moment().toJSON(); + const body = flatten(beatIds.map(beatId => [ + { update: { _id: `beat:${beatId}` } }, + { doc: { beat: { verified_on: verifiedOn } } } + ])); + + const params = { + index: INDEX_NAMES.BEATS, + type: '_doc', + body, + refresh: 'wait_for' + }; + + const response = await callWithRequest('bulk', params); + return get(response, 'items', []); +} + +function findNonExistentBeatIds(beatsFromEs, beatIdsFromRequest) { + return beatsFromEs.reduce((nonExistentBeatIds, beatFromEs, idx) => { + if (!beatFromEs.found) { + nonExistentBeatIds.push(beatIdsFromRequest[idx]); + } + return nonExistentBeatIds; + }, []); +} + +function findAlreadyVerifiedBeatIds(beatsFromEs) { + return beatsFromEs + .filter(beat => beat.found) + .filter(beat => beat._source.beat.hasOwnProperty('verified_on')) + .map(beat => beat._source.beat.id); +} + +function findToBeVerifiedBeatIds(beatsFromEs) { + return beatsFromEs + .filter(beat => beat.found) + .filter(beat => !beat._source.beat.hasOwnProperty('verified_on')) + .map(beat => beat._source.beat.id); +} + +function findVerifiedBeatIds(verifications, toBeVerifiedBeatIds) { + return verifications.reduce((verifiedBeatIds, verification, idx) => { + if (verification.update.status === 200) { + verifiedBeatIds.push(toBeVerifiedBeatIds[idx]); + } + return verifiedBeatIds; + }, []); +} + +// TODO: add license check pre-hook +// TODO: write to Kibana audit log file (include who did the verification as well) +export function registerVerifyBeatsRoute(server) { + server.route({ + method: 'POST', + path: '/api/beats/agents/verify', + config: { + validate: { + payload: Joi.object({ + beats: Joi.array({ + id: Joi.string().required() + }).min(1) + }).required() + } + }, + handler: async (request, reply) => { + const callWithRequest = callWithRequestFactory(server, request); + + const beats = [...request.payload.beats]; + const beatIds = beats.map(beat => beat.id); + + let nonExistentBeatIds; + let alreadyVerifiedBeatIds; + let verifiedBeatIds; + + try { + const beatsFromEs = await getBeats(callWithRequest, beatIds); + + nonExistentBeatIds = findNonExistentBeatIds(beatsFromEs, beatIds); + alreadyVerifiedBeatIds = findAlreadyVerifiedBeatIds(beatsFromEs); + const toBeVerifiedBeatIds = findToBeVerifiedBeatIds(beatsFromEs); + + const verifications = await verifyBeats(callWithRequest, toBeVerifiedBeatIds); + verifiedBeatIds = findVerifiedBeatIds(verifications, toBeVerifiedBeatIds); + + } catch (err) { + return reply(wrapEsError(err)); + } + + beats.forEach(beat => { + if (nonExistentBeatIds.includes(beat.id)) { + beat.status = 404; + beat.result = 'not found'; + } else if (alreadyVerifiedBeatIds.includes(beat.id)) { + beat.status = 200; + beat.result = 'already verified'; + } else if (verifiedBeatIds.includes(beat.id)) { + beat.status = 200; + beat.result = 'verified'; + } else { + beat.status = 400; + beat.result = 'not verified'; + } + }); + + const response = { beats }; + reply(response); + } + }); +} diff --git a/x-pack/test/api_integration/apis/beats/index.js b/x-pack/test/api_integration/apis/beats/index.js index 6b3562863a2b7..abb97b3daed91 100644 --- a/x-pack/test/api_integration/apis/beats/index.js +++ b/x-pack/test/api_integration/apis/beats/index.js @@ -20,5 +20,6 @@ export default function ({ getService, loadTestFile }) { loadTestFile(require.resolve('./create_enrollment_tokens')); loadTestFile(require.resolve('./enroll_beat')); loadTestFile(require.resolve('./list_beats')); + loadTestFile(require.resolve('./verify_beats')); }); } diff --git a/x-pack/test/api_integration/apis/beats/verify_beats.js b/x-pack/test/api_integration/apis/beats/verify_beats.js new file mode 100644 index 0000000000000..2b085308b43d1 --- /dev/null +++ b/x-pack/test/api_integration/apis/beats/verify_beats.js @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from 'expect.js'; + +export default function ({ getService }) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const chance = getService('chance'); + + describe('verify_beats', () => { + const archive = 'beats/list'; + + beforeEach('load beats archive', () => esArchiver.load(archive)); + afterEach('unload beats archive', () => esArchiver.unload(archive)); + + it('verify the given beats', async () => { + const { body: apiResponse } = await supertest + .post( + '/api/beats/agents/verify' + ) + .set('kbn-xsrf', 'xxx') + .send({ + beats: [ + { id: 'bar' }, + { id: 'baz' } + ] + }) + .expect(200); + + expect(apiResponse.beats).to.eql([ + { id: 'bar', status: 200, result: 'verified' }, + { id: 'baz', status: 200, result: 'verified' }, + ]); + }); + + it('should not re-verify already-verified beats', async () => { + const { body: apiResponse } = await supertest + .post( + '/api/beats/agents/verify' + ) + .set('kbn-xsrf', 'xxx') + .send({ + beats: [ + { id: 'foo' }, + { id: 'bar' } + ] + }) + .expect(200); + + expect(apiResponse.beats).to.eql([ + { id: 'foo', status: 200, result: 'already verified' }, + { id: 'bar', status: 200, result: 'verified' } + ]); + }); + + it('should return errors for non-existent beats', async () => { + const nonExistentBeatId = chance.word(); + const { body: apiResponse } = await supertest + .post( + '/api/beats/agents/verify' + ) + .set('kbn-xsrf', 'xxx') + .send({ + beats: [ + { id: 'bar' }, + { id: nonExistentBeatId } + ] + }) + .expect(200); + + expect(apiResponse.beats).to.eql([ + { id: 'bar', status: 200, result: 'verified' }, + { id: nonExistentBeatId, status: 404, result: 'not found' }, + ]); + }); + }); +} diff --git a/x-pack/test/functional/es_archives/beats/list/data.json.gz b/x-pack/test/functional/es_archives/beats/list/data.json.gz index c5bcfc6fb14f91eaed3dd09d5fc8ec099ff637a3..f3ccd2687455692022cc5fb6622b906cd0ec6293 100644 GIT binary patch literal 371 zcmV-(0gV11iwFp7ua17u-zVJ>QOZ*Bm^RKae8Fbuu(6^Qd1C6J^E-?7s!$VtqG zHb@7w>Q?pNM`$aU)^>+Y?In`!_pGO9JG&^3lm26cNggN8+vFi6Ht@C%ncWZ!VbwU? z1^}s{foH6-=@$l}??(8nLvd;mST1A&EPr2bPub3|TRZihaRc&*ijUERn&Hao4ZmTB z+Kcb{qFRMABPq!U|50tAKG3}<23lf$J;xl>PC~~dSc_d(^!^o_P}U%=)_}BhiW@4G zVtPp#liE53+$2ZpK03YoXdgwpo0x3i^Z!h)v2QDT#pZNyIU|e_e%b0l(PgVAxo53r zN~J=e5t0JuT$yDmg|q^-wt%v{tJT8}-O%bkZS*Ad{6=S%19y%pA-QEr!xAk=UQ#ZN zUWz$)gKbq-=n6klQ_9qWiUkvoOy;S`GevaDpYD7F?d^V=VKCzrTseU-n+xmTUYpA= RW6|eL{sI(FHM`CQ003wLvN`|& literal 343 zcmV-d0jT~TiwFqyEc;pj17u-zVJ>QOZ*Bm^Qp;|GFc7@+6^L_V*QP zHQ*FTls3x07v~|UN_uIPUM%fAR-^GAqBu^5_YEdRoH%cjhXCwgy$#4=9LBM39qxmG zG|<8`HrNg;gD~_b`D{aZT@hR^AVF5VZTDBS_uI}+yJy~@yr|;KG^u8~s$Sz4?a00O zekkirpczR?M))_jh30Jco*3we_03#!PCErXfnY86eL477Yy+)9TCD+T7VT>oK~%$LJVEhr5();N$N~ZgA*o`$Ns?*m6b~Bm8#NW1`ztPjMHkW=f?( zpb?S+=UkaQl|ov9T3bL_{cF|Z4c)QoUtRPRb@`$*%Yi#bm5|&rr6EVlkyn&UqjNF$ p?y#$?8eQp6)|4`}qGH9wBa=lcicArm@~7pW`2>YU+a_rQ0061trn&$C