From 5886ebd3f87111de10a42f537b90547dc4bfa4f8 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 23 May 2018 11:22:40 -0700 Subject: [PATCH] [Beats Management] APIs: Create or update tag (#19342) * Updating mappings * Implementing PUT /api/beats/tag/{tag} API --- .../lib/index_template/beats_template.json | 22 +- .../plugins/beats/server/routes/api/index.js | 2 + .../routes/api/register_set_tag_route.js | 124 +++++++++++ .../test/api_integration/apis/beats/index.js | 1 + .../api_integration/apis/beats/set_tag.js | 207 ++++++++++++++++++ .../es_archives/beats/list/mappings.json | 22 +- 6 files changed, 364 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugins/beats/server/routes/api/register_set_tag_route.js create mode 100644 x-pack/test/api_integration/apis/beats/set_tag.js diff --git a/x-pack/plugins/beats/server/lib/index_template/beats_template.json b/x-pack/plugins/beats/server/lib/index_template/beats_template.json index e293845f9a3a8..9f912f19b2a8d 100644 --- a/x-pack/plugins/beats/server/lib/index_template/beats_template.json +++ b/x-pack/plugins/beats/server/lib/index_template/beats_template.json @@ -27,16 +27,21 @@ } } }, - "configuration_block": { + "tag": { "properties": { - "tag": { - "type": "keyword" - }, - "type": { + "id": { "type": "keyword" }, - "block_yml": { - "type": "text" + "configuration_blocks": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "block_yml": { + "type": "text" + } + } } } }, @@ -69,6 +74,9 @@ "local_configuration_yml": { "type": "text" }, + "tags": { + "type": "keyword" + }, "central_configuration_yml": { "type": "text" }, diff --git a/x-pack/plugins/beats/server/routes/api/index.js b/x-pack/plugins/beats/server/routes/api/index.js index 4e6ee318668bf..5d7570807d682 100644 --- a/x-pack/plugins/beats/server/routes/api/index.js +++ b/x-pack/plugins/beats/server/routes/api/index.js @@ -9,6 +9,7 @@ import { registerEnrollBeatRoute } from './register_enroll_beat_route'; import { registerListBeatsRoute } from './register_list_beats_route'; import { registerVerifyBeatsRoute } from './register_verify_beats_route'; import { registerUpdateBeatRoute } from './register_update_beat_route'; +import { registerSetTagRoute } from './register_set_tag_route'; export function registerApiRoutes(server) { registerCreateEnrollmentTokensRoute(server); @@ -16,4 +17,5 @@ export function registerApiRoutes(server) { registerListBeatsRoute(server); registerVerifyBeatsRoute(server); registerUpdateBeatRoute(server); + registerSetTagRoute(server); } diff --git a/x-pack/plugins/beats/server/routes/api/register_set_tag_route.js b/x-pack/plugins/beats/server/routes/api/register_set_tag_route.js new file mode 100644 index 0000000000000..288fcade9929b --- /dev/null +++ b/x-pack/plugins/beats/server/routes/api/register_set_tag_route.js @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Joi from 'joi'; +import { + get, + uniq, + intersection +} from 'lodash'; +import { + INDEX_NAMES, + CONFIGURATION_BLOCKS +} from '../../../common/constants'; +import { callWithRequestFactory } from '../../lib/client'; +import { wrapEsError } from '../../lib/error_wrappers'; + +function validateUniquenessEnforcingTypes(configurationBlocks) { + const types = uniq(configurationBlocks.map(block => block.type)); + + // If none of the types in the given configuration blocks are uniqueness-enforcing, + // we don't need to perform any further validation checks. + const uniquenessEnforcingTypes = intersection(types, CONFIGURATION_BLOCKS.UNIQUENESS_ENFORCING_TYPES); + if (uniquenessEnforcingTypes.length === 0) { + return { isValid: true }; + } + + // Count the number of uniqueness-enforcing types in the given configuration blocks + const typeCountMap = configurationBlocks.reduce((typeCountMap, block) => { + const { type } = block; + if (!uniquenessEnforcingTypes.includes(type)) { + return typeCountMap; + } + + const count = typeCountMap[type] || 0; + return { + ...typeCountMap, + [type]: count + 1 + }; + }, {}); + + // If there is no more than one of any uniqueness-enforcing types in the given + // configuration blocks, we don't need to perform any further validation checks. + if (Object.values(typeCountMap).filter(count => count > 1).length === 0) { + return { isValid: true }; + } + + const message = Object.entries(typeCountMap) + .filter(([, count]) => count > 1) + .map(([type, count]) => `Expected only one configuration block of type '${type}' but found ${count}`) + .join(' '); + + return { + isValid: false, + message + }; +} + +async function validateConfigurationBlocks(configurationBlocks) { + return validateUniquenessEnforcingTypes(configurationBlocks); +} + +async function persistTag(callWithRequest, tag) { + const body = { + type: 'tag', + tag + }; + + const params = { + index: INDEX_NAMES.BEATS, + type: '_doc', + id: `tag:${tag.id}`, + body, + refresh: 'wait_for' + }; + + const response = await callWithRequest('index', params); + return response.result; +} + +// TODO: add license check pre-hook +// TODO: write to Kibana audit log file +export function registerSetTagRoute(server) { + server.route({ + method: 'PUT', + path: '/api/beats/tag/{tag}', + config: { + validate: { + payload: Joi.object({ + configuration_blocks: Joi.array().items( + Joi.object({ + type: Joi.string().required().valid(Object.values(CONFIGURATION_BLOCKS.TYPES)), + block_yml: Joi.string().required() + }) + ) + }).allow(null) + } + }, + handler: async (request, reply) => { + const callWithRequest = callWithRequestFactory(server, request); + + let result; + try { + const configurationBlocks = get(request, 'payload.configuration_blocks', []); + const { isValid, message } = await validateConfigurationBlocks(configurationBlocks); + if (!isValid) { + return reply({ message }).code(400); + } + + const tag = { + id: request.params.tag, + configuration_blocks: configurationBlocks + }; + result = await persistTag(callWithRequest, tag); + } catch (err) { + return reply(wrapEsError(err)); + } + + reply().code(result === 'created' ? 201 : 200); + } + }); +} diff --git a/x-pack/test/api_integration/apis/beats/index.js b/x-pack/test/api_integration/apis/beats/index.js index b41f17ed749b3..c3f07ecaa2926 100644 --- a/x-pack/test/api_integration/apis/beats/index.js +++ b/x-pack/test/api_integration/apis/beats/index.js @@ -22,5 +22,6 @@ export default function ({ getService, loadTestFile }) { loadTestFile(require.resolve('./list_beats')); loadTestFile(require.resolve('./verify_beats')); loadTestFile(require.resolve('./update_beat')); + loadTestFile(require.resolve('./set_tag')); }); } diff --git a/x-pack/test/api_integration/apis/beats/set_tag.js b/x-pack/test/api_integration/apis/beats/set_tag.js new file mode 100644 index 0000000000000..3af3c0372e847 --- /dev/null +++ b/x-pack/test/api_integration/apis/beats/set_tag.js @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from 'expect.js'; +import { + ES_INDEX_NAME, + ES_TYPE_NAME +} from './constants'; + +export default function ({ getService }) { + const supertest = getService('supertest'); + const chance = getService('chance'); + const es = getService('es'); + + describe('set_tag', () => { + it('should create an empty tag', async () => { + const tagId = 'production'; + await supertest + .put( + `/api/beats/tag/${tagId}` + ) + .set('kbn-xsrf', 'xxx') + .send() + .expect(201); + + const esResponse = await es.get({ + index: ES_INDEX_NAME, + type: ES_TYPE_NAME, + id: `tag:${tagId}` + }); + + const tagInEs = esResponse._source; + + expect(tagInEs.type).to.be('tag'); + expect(tagInEs.tag.id).to.be(tagId); + expect(tagInEs.tag.configuration_blocks).to.be.an(Array); + expect(tagInEs.tag.configuration_blocks.length).to.be(0); + }); + + it('should create a tag with one configuration block', async () => { + const tagId = 'production'; + await supertest + .put( + `/api/beats/tag/${tagId}` + ) + .set('kbn-xsrf', 'xxx') + .send({ + configuration_blocks: [ + { + type: 'output', + block_yml: 'elasticsearch:\n hosts: [\"localhost:9200\"]\n username: "..."' + } + ] + }) + .expect(201); + + const esResponse = await es.get({ + index: ES_INDEX_NAME, + type: ES_TYPE_NAME, + id: `tag:${tagId}` + }); + + const tagInEs = esResponse._source; + + expect(tagInEs.type).to.be('tag'); + expect(tagInEs.tag.id).to.be(tagId); + expect(tagInEs.tag.configuration_blocks).to.be.an(Array); + expect(tagInEs.tag.configuration_blocks.length).to.be(1); + expect(tagInEs.tag.configuration_blocks[0].type).to.be('output'); + expect(tagInEs.tag.configuration_blocks[0].block_yml).to.be('elasticsearch:\n hosts: [\"localhost:9200\"]\n username: "..."'); + }); + + it('should create a tag with two configuration blocks', async () => { + const tagId = 'production'; + await supertest + .put( + `/api/beats/tag/${tagId}` + ) + .set('kbn-xsrf', 'xxx') + .send({ + configuration_blocks: [ + { + type: 'filebeat.inputs', + block_yml: 'file:\n path: "/var/log/some.log"]\n' + }, + { + type: 'output', + block_yml: 'elasticsearch:\n hosts: [\"localhost:9200\"]\n username: "..."' + } + ] + }) + .expect(201); + + const esResponse = await es.get({ + index: ES_INDEX_NAME, + type: ES_TYPE_NAME, + id: `tag:${tagId}` + }); + + const tagInEs = esResponse._source; + + expect(tagInEs.type).to.be('tag'); + expect(tagInEs.tag.id).to.be(tagId); + expect(tagInEs.tag.configuration_blocks).to.be.an(Array); + expect(tagInEs.tag.configuration_blocks.length).to.be(2); + expect(tagInEs.tag.configuration_blocks[0].type).to.be('filebeat.inputs'); + expect(tagInEs.tag.configuration_blocks[0].block_yml).to.be('file:\n path: "/var/log/some.log"]\n'); + expect(tagInEs.tag.configuration_blocks[1].type).to.be('output'); + expect(tagInEs.tag.configuration_blocks[1].block_yml).to.be('elasticsearch:\n hosts: [\"localhost:9200\"]\n username: "..."'); + }); + + it('should fail when creating a tag with two configuration blocks of type output', async () => { + const tagId = 'production'; + await supertest + .put( + `/api/beats/tag/${tagId}` + ) + .set('kbn-xsrf', 'xxx') + .send({ + configuration_blocks: [ + { + type: 'output', + block_yml: 'logstash:\n hosts: ["localhost:9000"]\n' + }, + { + type: 'output', + block_yml: 'elasticsearch:\n hosts: [\"localhost:9200\"]\n username: "..."' + } + ] + }) + .expect(400); + }); + + it('should fail when creating a tag with an invalid configuration block type', async () => { + const tagId = 'production'; + await supertest + .put( + `/api/beats/tag/${tagId}` + ) + .set('kbn-xsrf', 'xxx') + .send({ + configuration_blocks: [ + { + type: chance.word(), + block_yml: 'logstash:\n hosts: ["localhost:9000"]\n' + } + ] + }) + .expect(400); + }); + + it('should update an existing tag', async () => { + const tagId = 'production'; + await supertest + .put( + `/api/beats/tag/${tagId}` + ) + .set('kbn-xsrf', 'xxx') + .send({ + configuration_blocks: [ + { + type: 'filebeat.inputs', + block_yml: 'file:\n path: "/var/log/some.log"]\n' + }, + { + type: 'output', + block_yml: 'elasticsearch:\n hosts: [\"localhost:9200\"]\n username: "..."' + } + ] + }) + .expect(201); + + await supertest + .put( + `/api/beats/tag/${tagId}` + ) + .set('kbn-xsrf', 'xxx') + .send({ + configuration_blocks: [ + { + type: 'output', + block_yml: 'logstash:\n hosts: ["localhost:9000"]\n' + } + ] + }) + .expect(200); + + const esResponse = await es.get({ + index: ES_INDEX_NAME, + type: ES_TYPE_NAME, + id: `tag:${tagId}` + }); + + const tagInEs = esResponse._source; + + expect(tagInEs.type).to.be('tag'); + expect(tagInEs.tag.id).to.be(tagId); + expect(tagInEs.tag.configuration_blocks).to.be.an(Array); + expect(tagInEs.tag.configuration_blocks.length).to.be(1); + expect(tagInEs.tag.configuration_blocks[0].type).to.be('output'); + expect(tagInEs.tag.configuration_blocks[0].block_yml).to.be('logstash:\n hosts: ["localhost:9000"]\n'); + }); + }); +} diff --git a/x-pack/test/functional/es_archives/beats/list/mappings.json b/x-pack/test/functional/es_archives/beats/list/mappings.json index 24aceed8a5b61..0057b0b765773 100644 --- a/x-pack/test/functional/es_archives/beats/list/mappings.json +++ b/x-pack/test/functional/es_archives/beats/list/mappings.json @@ -27,16 +27,21 @@ } } }, - "configuration_block": { + "tag": { "properties": { - "tag": { - "type": "keyword" - }, - "type": { + "id": { "type": "keyword" }, - "block_yml": { - "type": "text" + "configuration_blocks": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "block_yml": { + "type": "text" + } + } } } }, @@ -69,6 +74,9 @@ "local_configuration_yml": { "type": "text" }, + "tags": { + "type": "keyword" + }, "central_configuration_yml": { "type": "text" },