[Beats Management] APIs: Verify beats (#19103)

* WIP checkin

* WIP checkin

* Add API integration test

* Converting to Jest test

* Fixing API for default case + adding test for it

* Fixing copy pasta typos

* Fixing variable name

* Using a single index

* Implementing GET /api/beats/agents API

* Creating POST /api/beats/agents/verify API

* Refactoring: extracting out helper functions

* Fleshing out remaining tests

* Expanding TODO note so I won't forget :)

* Fixing file name

* Updating mapping

* Moving TODO comment to right file

* Rename determine* helper functions to find*

x-pack/plugins/beats/server/routes/api/index.js

@@ -7,9 +7,11 @@
 import { registerCreateEnrollmentTokensRoute } from './register_create_enrollment_tokens_route';
 import { registerEnrollBeatRoute } from './register_enroll_beat_route';
 import { registerListBeatsRoute } from './register_list_beats_route';
+import { registerVerifyBeatsRoute } from './register_verify_beats_route';
 
 export function registerApiRoutes(server) {
   registerCreateEnrollmentTokensRoute(server);
   registerEnrollBeatRoute(server);
   registerListBeatsRoute(server);
+  registerVerifyBeatsRoute(server);
 }

x-pack/plugins/beats/server/routes/api/register_verify_beats_route.js

@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import Joi from 'joi';
+import moment from 'moment';
+import {
+  get,
+  flatten
+} from 'lodash';
+import { INDEX_NAMES } from '../../../common/constants';
+import { callWithRequestFactory } from '../../lib/client';
+import { wrapEsError } from '../../lib/error_wrappers';
+
+async function getBeats(callWithRequest, beatIds) {
+  const ids = beatIds.map(beatId => `beat:${beatId}`);
+  const params = {
+    index: INDEX_NAMES.BEATS,
+    type: '_doc',
+    body: { ids },
+    _sourceInclude: [ 'beat.id', 'beat.verified_on' ]
+  };
+
+  const response = await callWithRequest('mget', params);
+  return get(response, 'docs', []);
+}
+
+async function verifyBeats(callWithRequest, beatIds) {
+  if (!Array.isArray(beatIds) || (beatIds.length === 0)) {
+    return [];
+  }
+
+  const verifiedOn = moment().toJSON();
+  const body = flatten(beatIds.map(beatId => [
+    { update: { _id: `beat:${beatId}` } },
+    { doc: { beat: { verified_on: verifiedOn } } }
+  ]));
+
+  const params = {
+    index: INDEX_NAMES.BEATS,
+    type: '_doc',
+    body,
+    refresh: 'wait_for'
+  };
+
+  const response = await callWithRequest('bulk', params);
+  return get(response, 'items', []);
+}
+
+function findNonExistentBeatIds(beatsFromEs, beatIdsFromRequest) {
+  return beatsFromEs.reduce((nonExistentBeatIds, beatFromEs, idx) => {
+    if (!beatFromEs.found) {
+      nonExistentBeatIds.push(beatIdsFromRequest[idx]);
+    }
+    return nonExistentBeatIds;
+  }, []);
+}
+
+function findAlreadyVerifiedBeatIds(beatsFromEs) {
+  return beatsFromEs
+    .filter(beat => beat.found)
+    .filter(beat => beat._source.beat.hasOwnProperty('verified_on'))
+    .map(beat => beat._source.beat.id);
+}
+
+function findToBeVerifiedBeatIds(beatsFromEs) {
+  return beatsFromEs
+    .filter(beat => beat.found)
+    .filter(beat => !beat._source.beat.hasOwnProperty('verified_on'))
+    .map(beat => beat._source.beat.id);
+}
+
+function findVerifiedBeatIds(verifications, toBeVerifiedBeatIds) {
+  return verifications.reduce((verifiedBeatIds, verification, idx) => {
+    if (verification.update.status === 200) {
+      verifiedBeatIds.push(toBeVerifiedBeatIds[idx]);
+    }
+    return verifiedBeatIds;
+  }, []);
+}
+
+// TODO: add license check pre-hook
+// TODO: write to Kibana audit log file (include who did the verification as well)
+export function registerVerifyBeatsRoute(server) {
+  server.route({
+    method: 'POST',
+    path: '/api/beats/agents/verify',
+    config: {
+      validate: {
+        payload: Joi.object({
+          beats: Joi.array({
+            id: Joi.string().required()
+          }).min(1)
+        }).required()
+      }
+    },
+    handler: async (request, reply) => {
+      const callWithRequest = callWithRequestFactory(server, request);
+
+      const beats = [...request.payload.beats];
+      const beatIds = beats.map(beat => beat.id);
+
+      let nonExistentBeatIds;
+      let alreadyVerifiedBeatIds;
+      let verifiedBeatIds;
+
+      try {
+        const beatsFromEs = await getBeats(callWithRequest, beatIds);
+
+        nonExistentBeatIds = findNonExistentBeatIds(beatsFromEs, beatIds);
+        alreadyVerifiedBeatIds = findAlreadyVerifiedBeatIds(beatsFromEs);
+        const toBeVerifiedBeatIds = findToBeVerifiedBeatIds(beatsFromEs);
+
+        const verifications = await verifyBeats(callWithRequest, toBeVerifiedBeatIds);
+        verifiedBeatIds = findVerifiedBeatIds(verifications, toBeVerifiedBeatIds);
+
+      } catch (err) {
+        return reply(wrapEsError(err));
+      }
+
+      beats.forEach(beat => {
+        if (nonExistentBeatIds.includes(beat.id)) {
+          beat.status = 404;
+          beat.result = 'not found';
+        } else if (alreadyVerifiedBeatIds.includes(beat.id)) {
+          beat.status = 200;
+          beat.result = 'already verified';
+        } else if (verifiedBeatIds.includes(beat.id)) {
+          beat.status = 200;
+          beat.result = 'verified';
+        } else {
+          beat.status = 400;
+          beat.result = 'not verified';
+        }
+      });
+
+      const response = { beats };
+      reply(response);
+    }
+  });
+}

x-pack/test/api_integration/apis/beats/index.js

@@ -20,5 +20,6 @@ export default function ({ getService, loadTestFile }) {
     loadTestFile(require.resolve('./create_enrollment_tokens'));
     loadTestFile(require.resolve('./enroll_beat'));
     loadTestFile(require.resolve('./list_beats'));
+    loadTestFile(require.resolve('./verify_beats'));
   });
 }

x-pack/test/api_integration/apis/beats/verify_beats.js

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from 'expect.js';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+  const chance = getService('chance');
+
+  describe('verify_beats', () => {
+    const archive = 'beats/list';
+
+    beforeEach('load beats archive', () => esArchiver.load(archive));
+    afterEach('unload beats archive', () => esArchiver.unload(archive));
+
+    it('verify the given beats', async () => {
+      const { body: apiResponse } = await supertest
+        .post(
+          '/api/beats/agents/verify'
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          beats: [
+            { id: 'bar' },
+            { id: 'baz' }
+          ]
+        })
+        .expect(200);
+
+      expect(apiResponse.beats).to.eql([
+        { id: 'bar', status: 200, result: 'verified' },
+        { id: 'baz', status: 200, result: 'verified' },
+      ]);
+    });
+
+    it('should not re-verify already-verified beats', async () => {
+      const { body: apiResponse } = await supertest
+        .post(
+          '/api/beats/agents/verify'
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          beats: [
+            { id: 'foo' },
+            { id: 'bar' }
+          ]
+        })
+        .expect(200);
+
+      expect(apiResponse.beats).to.eql([
+        { id: 'foo', status: 200, result: 'already verified' },
+        { id: 'bar', status: 200, result: 'verified' }
+      ]);
+    });
+
+    it('should return errors for non-existent beats', async () => {
+      const nonExistentBeatId = chance.word();
+      const { body: apiResponse } = await supertest
+        .post(
+          '/api/beats/agents/verify'
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          beats: [
+            { id: 'bar' },
+            { id: nonExistentBeatId }
+          ]
+        })
+        .expect(200);
+
+      expect(apiResponse.beats).to.eql([
+        { id: 'bar', status: 200, result: 'verified' },
+        { id: nonExistentBeatId, status: 404, result: 'not found' },
+      ]);
+    });
+  });
+}