From a35aac2c5d422705f01dee4c687e86fa027adb8b Mon Sep 17 00:00:00 2001 From: Jim Unger Date: Wed, 31 Aug 2016 12:36:55 -0500 Subject: [PATCH] Removed pipelines from ingest endpoint, and import csv wizard --- .../upload_data_step/upload_data_step.js | 3 +- .../filebeat/directives/filebeat_wizard.js | 1 - .../schemas/resources/ingest_config_schema.js | 4 +- .../server/routes/api/ingest/register_post.js | 43 +------------------ src/ui/public/ingest/__tests__/ingest.js | 39 ++--------------- src/ui/public/ingest/ingest.js | 42 +----------------- test/unit/api/ingest/_post.js | 29 ------------- test/unit/api/ingest/data.js | 8 +--- 8 files changed, 10 insertions(+), 159 deletions(-) diff --git a/src/core_plugins/kibana/public/management/sections/indices/add_data_steps/upload_data_step/upload_data_step.js b/src/core_plugins/kibana/public/management/sections/indices/add_data_steps/upload_data_step/upload_data_step.js index 768ef88dd8f9b..7e4fe54e76e78 100644 --- a/src/core_plugins/kibana/public/management/sections/indices/add_data_steps/upload_data_step/upload_data_step.js +++ b/src/core_plugins/kibana/public/management/sections/indices/add_data_steps/upload_data_step/upload_data_step.js @@ -27,8 +27,7 @@ modules.get('apps/management') location: 'Add Data' }); - const usePipeline = !_.isEmpty(_.get(this.results, 'pipeline.processors')); - ingest.uploadCSV(this.results.file, this.results.indexPattern.id, this.results.parseOptions.delimiter, usePipeline) + ingest.uploadCSV(this.results.file, this.results.indexPattern.id, this.results.parseOptions.delimiter) .then( (res) => { this.created = 0; diff --git a/src/core_plugins/kibana/public/management/sections/indices/filebeat/directives/filebeat_wizard.js b/src/core_plugins/kibana/public/management/sections/indices/filebeat/directives/filebeat_wizard.js index a454cd8d88349..f6014b86d0dd8 100644 --- a/src/core_plugins/kibana/public/management/sections/indices/filebeat/directives/filebeat_wizard.js +++ b/src/core_plugins/kibana/public/management/sections/indices/filebeat/directives/filebeat_wizard.js @@ -3,7 +3,6 @@ import template from 'plugins/kibana/management/sections/indices/filebeat/direct import IngestProvider from 'ui/ingest'; import 'plugins/kibana/management/sections/indices/add_data_steps/pattern_review_step'; import 'plugins/kibana/management/sections/indices/add_data_steps/paste_samples_step'; -import 'plugins/kibana/management/sections/indices/add_data_steps/pipeline_setup'; import 'plugins/kibana/management/sections/indices/add_data_steps/install_filebeat_step'; import '../../styles/_add_data_wizard.less'; diff --git a/src/core_plugins/kibana/server/lib/schemas/resources/ingest_config_schema.js b/src/core_plugins/kibana/server/lib/schemas/resources/ingest_config_schema.js index 190a6be715b2e..f2bd7a4cac50e 100644 --- a/src/core_plugins/kibana/server/lib/schemas/resources/ingest_config_schema.js +++ b/src/core_plugins/kibana/server/lib/schemas/resources/ingest_config_schema.js @@ -1,8 +1,6 @@ import Joi from 'joi'; import indexPatternSchema from './index_pattern_schema'; -import pipelineSchema from './pipeline_schema'; module.exports = Joi.object({ - index_pattern: indexPatternSchema.required(), - pipeline: pipelineSchema + index_pattern: indexPatternSchema.required() }); diff --git a/src/core_plugins/kibana/server/routes/api/ingest/register_post.js b/src/core_plugins/kibana/server/routes/api/ingest/register_post.js index caf4d23f05795..ba6e4372dadfe 100644 --- a/src/core_plugins/kibana/server/routes/api/ingest/register_post.js +++ b/src/core_plugins/kibana/server/routes/api/ingest/register_post.js @@ -6,7 +6,6 @@ import createMappingsFromPatternFields from '../../../lib/create_mappings_from_p import initDefaultFieldProps from '../../../lib/init_default_field_props'; import {ingestToPattern, patternToIngest} from '../../../../common/lib/convert_pattern_and_ingest_name'; import { keysToCamelCaseShallow } from '../../../../common/lib/case_conversion'; -import ingestPipelineApiKibanaToEsConverter from '../../../lib/converters/ingest_pipeline_api_kibana_to_es_converter'; export function registerPost(server) { const kibanaIndex = server.config().get('kibana.index'); @@ -25,7 +24,7 @@ export function registerPost(server) { }, (patternDeletionError) => { throw new Error( - `index-pattern ${indexPatternId} created successfully but index template or pipeline + `index-pattern ${indexPatternId} created successfully but index template creation failed. Failed to rollback index-pattern creation, must delete manually. ${patternDeletionError.toString()} ${rootError.toString()}` @@ -34,27 +33,6 @@ export function registerPost(server) { ); } - function templateRollback(rootError, templateName, boundCallWithRequest) { - const deleteParams = { - name: templateName - }; - - return boundCallWithRequest('indices.deleteTemplate', deleteParams) - .then( - () => { - throw rootError; - }, - (templateDeletionError) => { - throw new Error( - `index template ${templateName} created successfully but pipeline - creation failed. Failed to rollback template creation, must delete manually. - ${templateDeletionError.toString()} - ${rootError.toString()}` - ); - } - ); - } - server.route({ path: '/api/kibana/ingest', method: 'POST', @@ -71,7 +49,6 @@ export function registerPost(server) { const indexPattern = keysToCamelCaseShallow(requestDocument.index_pattern); const indexPatternId = indexPattern.id; const ingestConfigName = patternToIngest(indexPatternId); - const shouldCreatePipeline = !_.isEmpty(requestDocument.pipeline); delete indexPattern.id; const mappings = createMappingsFromPatternFields(indexPattern.fields); @@ -81,8 +58,6 @@ export function registerPost(server) { indexPattern.fields = JSON.stringify(indexPattern.fields); indexPattern.fieldFormatMap = JSON.stringify(indexPattern.fieldFormatMap); - const pipeline = ingestPipelineApiKibanaToEsConverter(requestDocument.pipeline); - // Set up call with request params const patternCreateParams = { index: kibanaIndex, @@ -105,13 +80,6 @@ export function registerPost(server) { } }; - const pipelineParams = { - path: `/_ingest/pipeline/${ingestConfigName}`, - method: 'PUT', - body: pipeline - }; - - return boundCallWithRequest('indices.exists', {index: indexPatternId}) .then((matchingIndices) => { if (matchingIndices) { @@ -122,15 +90,6 @@ export function registerPost(server) { .then(() => { return boundCallWithRequest('indices.putTemplate', templateParams) .catch((templateError) => {return patternRollback(templateError, indexPatternId, boundCallWithRequest);}); - }) - .then((templateResponse) => { - if (!shouldCreatePipeline) { - return templateResponse; - } - - return boundCallWithRequest('transport.request', pipelineParams) - .catch((pipelineError) => {return templateRollback(pipelineError, ingestConfigName, boundCallWithRequest);}) - .catch((templateRollbackError) => {return patternRollback(templateRollbackError, indexPatternId, boundCallWithRequest);}); }); }) .then( diff --git a/src/ui/public/ingest/__tests__/ingest.js b/src/ui/public/ingest/__tests__/ingest.js index 4ca53e5e9db3f..6bb5b96a316ab 100644 --- a/src/ui/public/ingest/__tests__/ingest.js +++ b/src/ui/public/ingest/__tests__/ingest.js @@ -122,7 +122,7 @@ describe('Ingest Service', function () { it('POSTs to the kibana _data endpoint with the correct params and the file attached as multipart/form-data', function () { $httpBackend - .expectPOST('/api/kibana/foo/_data?csv_delimiter=;&pipeline=true', function (data) { + .expectPOST('/api/kibana/foo/_data?csv_delimiter=;', function (data) { // The assertions we can do here are limited because of poor browser support for FormData methods return data instanceof FormData; }) @@ -130,18 +130,18 @@ describe('Ingest Service', function () { const file = new Blob(['foo,bar'], {type : 'text/csv'}); - ingest.uploadCSV(file, 'foo', ';', true); + ingest.uploadCSV(file, 'foo', ';'); $httpBackend.flush(); }); it('Returns error from the data API if there is one', function (done) { $httpBackend - .expectPOST('/api/kibana/foo/_data?csv_delimiter=;&pipeline=true') + .expectPOST('/api/kibana/foo/_data?csv_delimiter=;') .respond(404); const file = new Blob(['foo,bar'], {type : 'text/csv'}); - ingest.uploadCSV(file, 'foo', ';', true) + ingest.uploadCSV(file, 'foo', ';') .then( () => { throw new Error('expected an error response'); @@ -156,35 +156,4 @@ describe('Ingest Service', function () { }); }); - describe('getProcessors', () => { - - it('Calls the processors GET endpoint of the ingest API', function () { - $httpBackend - .expectGET('/api/kibana/ingest/processors') - .respond('ok'); - - ingest.getProcessors(); - $httpBackend.flush(); - }); - - it('Throws user-friendly error when there is an error in the request', function (done) { - $httpBackend - .when('GET', '/api/kibana/ingest/processors') - .respond(404); - - ingest.getProcessors() - .then( - () => { - throw new Error('expected an error response'); - }, - (error) => { - expect(error.message).to.be('Error fetching enabled processors'); - done(); - }); - - $httpBackend.flush(); - }); - - }); - }); diff --git a/src/ui/public/ingest/ingest.js b/src/ui/public/ingest/ingest.js index 47435dc9ce71d..d10765563f887 100644 --- a/src/ui/public/ingest/ingest.js +++ b/src/ui/public/ingest/ingest.js @@ -9,7 +9,7 @@ export default function IngestProvider($rootScope, $http, config, $q, Private, i const ingestAPIPrefix = chrome.addBasePath('/api/kibana/ingest'); const refreshKibanaIndex = Private(RefreshKibanaIndexProvider); - this.save = function (indexPattern, pipeline) { + this.save = function (indexPattern) { if (_.isEmpty(indexPattern)) { throw new Error('index pattern is required'); } @@ -17,9 +17,6 @@ export default function IngestProvider($rootScope, $http, config, $q, Private, i const payload = { index_pattern: keysToSnakeCaseShallow(indexPattern) }; - if (!_.isEmpty(pipeline)) { - payload.pipeline = _.map(pipeline, processor => keysToSnakeCaseShallow(processor)); - } return $http.post(`${ingestAPIPrefix}`, payload) .then(() => { @@ -44,39 +41,7 @@ export default function IngestProvider($rootScope, $http, config, $q, Private, i }); }; - this.simulate = function (pipeline) { - function pack(pipeline) { - const result = keysToSnakeCaseShallow(pipeline); - result.processors = _.map(result.processors, processor => keysToSnakeCaseShallow(processor)); - - return result; - } - - function unpack(response) { - const data = response.data.map(result => keysToCamelCaseShallow(result)); - return data; - } - - return $http.post(`${ingestAPIPrefix}/simulate`, pack(pipeline)) - .then(unpack) - .catch(err => { - return $q.reject(new Error('Error simulating pipeline')); - }); - }; - - this.getProcessors = function () { - function unpack(response) { - return response.data; - } - - return $http.get(`${ingestAPIPrefix}/processors`) - .then(unpack) - .catch(err => { - return $q.reject(new Error('Error fetching enabled processors')); - }); - }; - - this.uploadCSV = function (file, indexPattern, delimiter, pipeline) { + this.uploadCSV = function (file, indexPattern, delimiter) { if (_.isUndefined(file)) { throw new Error('file is required'); } @@ -91,9 +56,6 @@ export default function IngestProvider($rootScope, $http, config, $q, Private, i if (!_.isUndefined(delimiter)) { params.csv_delimiter = delimiter; } - if (!_.isUndefined(pipeline)) { - params.pipeline = pipeline; - } return $http.post(chrome.addBasePath(`/api/kibana/${indexPattern}/_data`), formData, { params: params, diff --git a/test/unit/api/ingest/_post.js b/test/unit/api/ingest/_post.js index 732c9499a70fe..c48b3bbed0b28 100644 --- a/test/unit/api/ingest/_post.js +++ b/test/unit/api/ingest/_post.js @@ -43,11 +43,6 @@ define(function (require) { // Fields must have a name and type request.post('/kibana/ingest') .send(_.set(createTestData(), 'index_pattern.fields', [{count: 0}])) - .expect(400), - - // should validate pipeline processors - request.post('/kibana/ingest') - .send(_.set(createTestData(), 'pipeline[0]', {bad: 'processor'})) .expect(400) ]); }); @@ -165,30 +160,6 @@ define(function (require) { }); }); - bdd.it('should create a pipeline if one is included in the request', function () { - return request.post('/kibana/ingest') - .send(createTestData()) - .expect(204) - .then(function () { - return scenarioManager.client.transport.request({ - path: '_ingest/pipeline/kibana-logstash-*', - method: 'GET' - }) - .then(function (body) { - expect(body).to.have.property('kibana-logstash-*'); - }); - }); - }); - - bdd.it('pipeline should be optional', function optionalPipeline() { - const payload = createTestData(); - delete payload.pipeline; - - return request.post('/kibana/ingest') - .send(payload) - .expect(204); - }); - bdd.it('should return 409 conflict when a pattern with the given ID already exists', function patternConflict() { return request.post('/kibana/ingest') .send(createTestData()) diff --git a/test/unit/api/ingest/data.js b/test/unit/api/ingest/data.js index 60a7389a8682b..b64c1f5007fbf 100644 --- a/test/unit/api/ingest/data.js +++ b/test/unit/api/ingest/data.js @@ -23,12 +23,6 @@ module.exports = function createTestData() { 'type': 'geo_point' } ] - }, - pipeline: [{ - processor_id: 'processor1', - type_id: 'set', - target_field: 'foo', - value: 'bar' - }] + } }; };