[Beats Management] APIs: Create or update tag (elastic#19342)

* Updating mappings

* Implementing PUT /api/beats/tag/{tag} API

x-pack/plugins/beats/server/lib/index_template/beats_template.json

@@ -27,16 +27,21 @@
             }
           }
         },
-        "configuration_block": {
+        "tag": {
           "properties": {
-            "tag": {
-              "type": "keyword"
-            },
-            "type": {
+            "id": {
               "type": "keyword"
             },
-            "block_yml": {
-              "type": "text"
+            "configuration_blocks": {
+              "type": "nested",
+              "properties": {
+                "type": {
+                  "type": "keyword"
+                },
+                "block_yml": {
+                  "type": "text"
+                }
+              }
             }
           }
         },
@@ -69,6 +74,9 @@
             "local_configuration_yml": {
               "type": "text"
             },
+            "tags": {
+              "type": "keyword"
+            },
             "central_configuration_yml": {
               "type": "text"
             },

x-pack/plugins/beats/server/routes/api/index.js

@@ -9,11 +9,13 @@ import { registerEnrollBeatRoute } from './register_enroll_beat_route';
 import { registerListBeatsRoute } from './register_list_beats_route';
 import { registerVerifyBeatsRoute } from './register_verify_beats_route';
 import { registerUpdateBeatRoute } from './register_update_beat_route';
+import { registerSetTagRoute } from './register_set_tag_route';
 
 export function registerApiRoutes(server) {
   registerCreateEnrollmentTokensRoute(server);
   registerEnrollBeatRoute(server);
   registerListBeatsRoute(server);
   registerVerifyBeatsRoute(server);
   registerUpdateBeatRoute(server);
+  registerSetTagRoute(server);
 }

x-pack/plugins/beats/server/routes/api/register_set_tag_route.js

@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import Joi from 'joi';
+import {
+  get,
+  uniq,
+  intersection
+} from 'lodash';
+import {
+  INDEX_NAMES,
+  CONFIGURATION_BLOCKS
+} from '../../../common/constants';
+import { callWithRequestFactory } from '../../lib/client';
+import { wrapEsError } from '../../lib/error_wrappers';
+
+function validateUniquenessEnforcingTypes(configurationBlocks) {
+  const types = uniq(configurationBlocks.map(block => block.type));
+
+  // If none of the types in the given configuration blocks are uniqueness-enforcing,
+  // we don't need to perform any further validation checks.
+  const uniquenessEnforcingTypes = intersection(types, CONFIGURATION_BLOCKS.UNIQUENESS_ENFORCING_TYPES);
+  if (uniquenessEnforcingTypes.length === 0) {
+    return { isValid: true };
+  }
+
+  // Count the number of uniqueness-enforcing types in the given configuration blocks
+  const typeCountMap = configurationBlocks.reduce((typeCountMap, block) => {
+    const { type } = block;
+    if (!uniquenessEnforcingTypes.includes(type)) {
+      return typeCountMap;
+    }
+
+    const count = typeCountMap[type] || 0;
+    return {
+      ...typeCountMap,
+      [type]: count + 1
+    };
+  }, {});
+
+  // If there is no more than one of any uniqueness-enforcing types in the given
+  // configuration blocks, we don't need to perform any further validation checks.
+  if (Object.values(typeCountMap).filter(count => count > 1).length === 0) {
+    return { isValid: true };
+  }
+
+  const message = Object.entries(typeCountMap)
+    .filter(([, count]) => count > 1)
+    .map(([type, count]) => `Expected only one configuration block of type '${type}' but found ${count}`)
+    .join(' ');
+
+  return {
+    isValid: false,
+    message
+  };
+}
+
+async function validateConfigurationBlocks(configurationBlocks) {
+  return validateUniquenessEnforcingTypes(configurationBlocks);
+}
+
+async function persistTag(callWithRequest, tag) {
+  const body = {
+    type: 'tag',
+    tag
+  };
+
+  const params = {
+    index: INDEX_NAMES.BEATS,
+    type: '_doc',
+    id: `tag:${tag.id}`,
+    body,
+    refresh: 'wait_for'
+  };
+
+  const response = await callWithRequest('index', params);
+  return response.result;
+}
+
+// TODO: add license check pre-hook
+// TODO: write to Kibana audit log file
+export function registerSetTagRoute(server) {
+  server.route({
+    method: 'PUT',
+    path: '/api/beats/tag/{tag}',
+    config: {
+      validate: {
+        payload: Joi.object({
+          configuration_blocks: Joi.array().items(
+            Joi.object({
+              type: Joi.string().required().valid(Object.values(CONFIGURATION_BLOCKS.TYPES)),
+              block_yml: Joi.string().required()
+            })
+          )
+        }).allow(null)
+      }
+    },
+    handler: async (request, reply) => {
+      const callWithRequest = callWithRequestFactory(server, request);
+
+      let result;
+      try {
+        const configurationBlocks = get(request, 'payload.configuration_blocks', []);
+        const { isValid, message } = await validateConfigurationBlocks(configurationBlocks);
+        if (!isValid) {
+          return reply({ message }).code(400);
+        }
+
+        const tag = {
+          id: request.params.tag,
+          configuration_blocks: configurationBlocks
+        };
+        result = await persistTag(callWithRequest, tag);
+      } catch (err) {
+        return reply(wrapEsError(err));
+      }
+
+      reply().code(result === 'created' ? 201 : 200);
+    }
+  });
+}

x-pack/test/api_integration/apis/beats/index.js

@@ -22,5 +22,6 @@ export default function ({ getService, loadTestFile }) {
     loadTestFile(require.resolve('./list_beats'));
     loadTestFile(require.resolve('./verify_beats'));
     loadTestFile(require.resolve('./update_beat'));
+    loadTestFile(require.resolve('./set_tag'));
   });
 }

x-pack/test/api_integration/apis/beats/set_tag.js

@@ -0,0 +1,207 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from 'expect.js';
+import {
+  ES_INDEX_NAME,
+  ES_TYPE_NAME
+} from './constants';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const chance = getService('chance');
+  const es = getService('es');
+
+  describe('set_tag', () => {
+    it('should create an empty tag', async () => {
+      const tagId = 'production';
+      await supertest
+        .put(
+          `/api/beats/tag/${tagId}`
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send()
+        .expect(201);
+
+      const esResponse = await es.get({
+        index: ES_INDEX_NAME,
+        type: ES_TYPE_NAME,
+        id: `tag:${tagId}`
+      });
+
+      const tagInEs = esResponse._source;
+
+      expect(tagInEs.type).to.be('tag');
+      expect(tagInEs.tag.id).to.be(tagId);
+      expect(tagInEs.tag.configuration_blocks).to.be.an(Array);
+      expect(tagInEs.tag.configuration_blocks.length).to.be(0);
+    });
+
+    it('should create a tag with one configuration block', async () => {
+      const tagId = 'production';
+      await supertest
+        .put(
+          `/api/beats/tag/${tagId}`
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          configuration_blocks: [
+            {
+              type: 'output',
+              block_yml: 'elasticsearch:\n    hosts: [\"localhost:9200\"]\n    username: "..."'
+            }
+          ]
+        })
+        .expect(201);
+
+      const esResponse = await es.get({
+        index: ES_INDEX_NAME,
+        type: ES_TYPE_NAME,
+        id: `tag:${tagId}`
+      });
+
+      const tagInEs = esResponse._source;
+
+      expect(tagInEs.type).to.be('tag');
+      expect(tagInEs.tag.id).to.be(tagId);
+      expect(tagInEs.tag.configuration_blocks).to.be.an(Array);
+      expect(tagInEs.tag.configuration_blocks.length).to.be(1);
+      expect(tagInEs.tag.configuration_blocks[0].type).to.be('output');
+      expect(tagInEs.tag.configuration_blocks[0].block_yml).to.be('elasticsearch:\n    hosts: [\"localhost:9200\"]\n    username: "..."');
+    });
+
+    it('should create a tag with two configuration blocks', async () => {
+      const tagId = 'production';
+      await supertest
+        .put(
+          `/api/beats/tag/${tagId}`
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          configuration_blocks: [
+            {
+              type: 'filebeat.inputs',
+              block_yml: 'file:\n    path: "/var/log/some.log"]\n'
+            },
+            {
+              type: 'output',
+              block_yml: 'elasticsearch:\n    hosts: [\"localhost:9200\"]\n    username: "..."'
+            }
+          ]
+        })
+        .expect(201);
+
+      const esResponse = await es.get({
+        index: ES_INDEX_NAME,
+        type: ES_TYPE_NAME,
+        id: `tag:${tagId}`
+      });
+
+      const tagInEs = esResponse._source;
+
+      expect(tagInEs.type).to.be('tag');
+      expect(tagInEs.tag.id).to.be(tagId);
+      expect(tagInEs.tag.configuration_blocks).to.be.an(Array);
+      expect(tagInEs.tag.configuration_blocks.length).to.be(2);
+      expect(tagInEs.tag.configuration_blocks[0].type).to.be('filebeat.inputs');
+      expect(tagInEs.tag.configuration_blocks[0].block_yml).to.be('file:\n    path: "/var/log/some.log"]\n');
+      expect(tagInEs.tag.configuration_blocks[1].type).to.be('output');
+      expect(tagInEs.tag.configuration_blocks[1].block_yml).to.be('elasticsearch:\n    hosts: [\"localhost:9200\"]\n    username: "..."');
+    });
+
+    it('should fail when creating a tag with two configuration blocks of type output', async () => {
+      const tagId = 'production';
+      await supertest
+        .put(
+          `/api/beats/tag/${tagId}`
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          configuration_blocks: [
+            {
+              type: 'output',
+              block_yml: 'logstash:\n    hosts: ["localhost:9000"]\n'
+            },
+            {
+              type: 'output',
+              block_yml: 'elasticsearch:\n    hosts: [\"localhost:9200\"]\n    username: "..."'
+            }
+          ]
+        })
+        .expect(400);
+    });
+
+    it('should fail when creating a tag with an invalid configuration block type', async () => {
+      const tagId = 'production';
+      await supertest
+        .put(
+          `/api/beats/tag/${tagId}`
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          configuration_blocks: [
+            {
+              type: chance.word(),
+              block_yml: 'logstash:\n    hosts: ["localhost:9000"]\n'
+            }
+          ]
+        })
+        .expect(400);
+    });
+
+    it('should update an existing tag', async () => {
+      const tagId = 'production';
+      await supertest
+        .put(
+          `/api/beats/tag/${tagId}`
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          configuration_blocks: [
+            {
+              type: 'filebeat.inputs',
+              block_yml: 'file:\n    path: "/var/log/some.log"]\n'
+            },
+            {
+              type: 'output',
+              block_yml: 'elasticsearch:\n    hosts: [\"localhost:9200\"]\n    username: "..."'
+            }
+          ]
+        })
+        .expect(201);
+
+      await supertest
+        .put(
+          `/api/beats/tag/${tagId}`
+        )
+        .set('kbn-xsrf', 'xxx')
+        .send({
+          configuration_blocks: [
+            {
+              type: 'output',
+              block_yml: 'logstash:\n    hosts: ["localhost:9000"]\n'
+            }
+          ]
+        })
+        .expect(200);
+
+      const esResponse = await es.get({
+        index: ES_INDEX_NAME,
+        type: ES_TYPE_NAME,
+        id: `tag:${tagId}`
+      });
+
+      const tagInEs = esResponse._source;
+
+      expect(tagInEs.type).to.be('tag');
+      expect(tagInEs.tag.id).to.be(tagId);
+      expect(tagInEs.tag.configuration_blocks).to.be.an(Array);
+      expect(tagInEs.tag.configuration_blocks.length).to.be(1);
+      expect(tagInEs.tag.configuration_blocks[0].type).to.be('output');
+      expect(tagInEs.tag.configuration_blocks[0].block_yml).to.be('logstash:\n    hosts: ["localhost:9000"]\n');
+    });
+  });
+}