diff --git a/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js b/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js index d95a41ae66d9a..1d4f29ab233e8 100644 --- a/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js +++ b/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js @@ -23,8 +23,9 @@ uiModules.get('apps/management') { title: 'name' }, { title: 'type' }, { title: 'format' }, + { title: 'searchable', info: 'These fields can be used in the filter bar' }, + { title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' }, { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' }, - { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' }, { title: 'controls', sortable: false } ]; @@ -54,12 +55,16 @@ uiModules.get('apps/management') }, _.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']), { - markup: field.analyzed ? yesTemplate : noTemplate, - value: field.analyzed + markup: field.searchable ? yesTemplate : noTemplate, + value: field.searchable + }, + { + markup: field.aggregatable ? yesTemplate : noTemplate, + value: field.aggregatable }, { - markup: field.indexed ? yesTemplate : noTemplate, - value: field.indexed + markup: field.analyzed ? yesTemplate : noTemplate, + value: field.analyzed }, { markup: controlsHtml, diff --git a/src/core_plugins/kibana/public/visualize/editor/agg_params.js b/src/core_plugins/kibana/public/visualize/editor/agg_params.js index a525057c017cd..81f40941305bf 100644 --- a/src/core_plugins/kibana/public/visualize/editor/agg_params.js +++ b/src/core_plugins/kibana/public/visualize/editor/agg_params.js @@ -138,7 +138,7 @@ uiModules } function getIndexedFields(param) { - let fields = $scope.agg.vis.indexPattern.fields.raw; + let fields = _.filter($scope.agg.vis.indexPattern.fields.raw, 'aggregatable'); const fieldTypes = param.filterFieldTypes; if (fieldTypes) { diff --git a/src/core_plugins/kibana/server/routes/api/ingest/index.js b/src/core_plugins/kibana/server/routes/api/ingest/index.js index 8b723c800946a..ff31b8104b76e 100644 --- a/src/core_plugins/kibana/server/routes/api/ingest/index.js +++ b/src/core_plugins/kibana/server/routes/api/ingest/index.js @@ -3,6 +3,7 @@ import { registerDelete } from './register_delete'; import { registerProcessors } from './register_processors'; import { registerSimulate } from './register_simulate'; import { registerData } from './register_data'; +import { registerFieldCapabilities } from './register_field_capabilities'; export default function (server) { registerPost(server); @@ -10,4 +11,5 @@ export default function (server) { registerProcessors(server); registerSimulate(server); registerData(server); + registerFieldCapabilities(server); } diff --git a/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js b/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js new file mode 100644 index 0000000000000..3a150804d748a --- /dev/null +++ b/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js @@ -0,0 +1,31 @@ +import _ from 'lodash'; +import handleESError from '../../../lib/handle_es_error'; + +export function registerFieldCapabilities(server) { + server.route({ + path: '/api/kibana/{indices}/field_capabilities', + method: ['GET'], + handler: function (req, reply) { + const callWithRequest = server.plugins.elasticsearch.callWithRequest; + const indices = req.params.indices || ''; + + return callWithRequest(req, 'fieldStats', { + fields: '*', + level: 'cluster', + index: indices, + allowNoIndices: false + }) + .catch((error) => { + reply(handleESError(error)); + }) + .then((res) => { + const fields = _.get(res, 'indices._all.fields', {}); + const fieldsFilteredValues = _.mapValues(fields, (value) => { + return _.pick(value, ['searchable', 'aggregatable']); + }); + + reply({fields: fieldsFilteredValues}); + }); + } + }); +} diff --git a/src/ui/public/index_patterns/__tests__/_index_pattern.js b/src/ui/public/index_patterns/__tests__/_index_pattern.js index 9503deeb9e446..2b6d7ca3d9e37 100644 --- a/src/ui/public/index_patterns/__tests__/_index_pattern.js +++ b/src/ui/public/index_patterns/__tests__/_index_pattern.js @@ -43,6 +43,9 @@ describe('index pattern', function () { sinon.stub(mapper, 'getFieldsForIndexPattern', function () { return Promise.resolve(_.filter(mockLogstashFields, { scripted: false })); }); + sinon.stub(mapper, 'clearCache', function () { + return Promise.resolve(); + }); // stub mappingSetup mappingSetup = Private(UtilsMappingSetupProvider); @@ -151,15 +154,34 @@ describe('index pattern', function () { const indexPatternId = 'test-pattern'; let indexPattern; let fieldLength; - let truncatedFields; + let customFields; beforeEach(function () { fieldLength = mockLogstashFields.length; - truncatedFields = mockLogstashFields.slice(3); + customFields = [{ + analyzed: true, + count: 30, + filterable: true, + indexed: true, + name: 'foo', + scripted: false, + sortable: true, + type: 'number', + aggregatable: true, + searchable: false + }, + { + name: 'script number', + type: 'number', + scripted: true, + script: '1234', + lang: 'expression' + }]; + return create(indexPatternId, { _source: { customFormats: '{}', - fields: JSON.stringify(truncatedFields) + fields: JSON.stringify(customFields) } }).then(function (pattern) { indexPattern = pattern; @@ -168,8 +190,8 @@ describe('index pattern', function () { it('should fetch fields from the doc source', function () { // ensure that we don't have all the fields - expect(truncatedFields.length).to.not.equal(mockLogstashFields.length); - expect(indexPattern.fields).to.have.length(truncatedFields.length); + expect(customFields.length).to.not.equal(mockLogstashFields.length); + expect(indexPattern.fields).to.have.length(customFields.length); // ensure that all fields will be included in the returned docSource setDocsourcePayload(docSourceResponse(indexPatternId)); @@ -201,9 +223,8 @@ describe('index pattern', function () { // called to append scripted fields to the response from mapper.getFieldsForIndexPattern expect(scriptedFieldsSpy.callCount).to.equal(1); - const scripted = _.where(mockLogstashFields, { scripted: true }); - const expected = _.filter(indexPattern.fields, { scripted: true }); - expect(_.pluck(expected, 'name')).to.eql(_.pluck(scripted, 'name')); + const expected = _.filter(indexPattern.fields, {scripted: true}); + expect(_.pluck(expected, 'name')).to.eql(['script number']); }); }); }); diff --git a/src/ui/public/index_patterns/_enhance_fields_with_capabilities.js b/src/ui/public/index_patterns/_enhance_fields_with_capabilities.js new file mode 100644 index 0000000000000..6bb1980fa799c --- /dev/null +++ b/src/ui/public/index_patterns/_enhance_fields_with_capabilities.js @@ -0,0 +1,16 @@ +import chrome from 'ui/chrome'; +import _ from 'lodash'; + +export default function ($http) { + + return function (fields, indices) { + return $http.get(chrome.addBasePath(`/api/kibana/${indices}/field_capabilities`)) + .then((res) => { + const stats = _.get(res, 'data.fields', {}); + + return _.map(fields, (field) => { + return _.assign(field, stats[field.name]); + }); + }); + }; +} diff --git a/src/ui/public/index_patterns/_field.js b/src/ui/public/index_patterns/_field.js index 986d1fa14ebdd..826a6da55a6ad 100644 --- a/src/ui/public/index_patterns/_field.js +++ b/src/ui/public/index_patterns/_field.js @@ -57,6 +57,10 @@ export default function FieldObjectProvider(Private, shortDotsFilter, $rootScope obj.fact('analyzed', !!spec.analyzed); obj.fact('doc_values', !!spec.doc_values); + // stats + obj.fact('searchable', !!spec.searchable); + obj.fact('aggregatable', !!spec.aggregatable); + // usage flags, read-only and won't be saved obj.comp('format', format); obj.comp('sortable', sortable); diff --git a/src/ui/public/index_patterns/_index_pattern.js b/src/ui/public/index_patterns/_index_pattern.js index 0f7b8c8130cee..6a088619ec407 100644 --- a/src/ui/public/index_patterns/_index_pattern.js +++ b/src/ui/public/index_patterns/_index_pattern.js @@ -82,23 +82,34 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex, // give index pattern all of the values in _source _.assign(indexPattern, response._source); - indexFields(indexPattern); + const promise = indexFields(indexPattern); // any time index pattern in ES is updated, update index pattern object docSources .get(indexPattern) .onUpdate() .then(response => updateFromElasticSearch(indexPattern, response), notify.fatal); + + return promise; + } + + function containsFieldCapabilities(fields) { + return _.any(fields, (field) => { + return _.has(field, 'aggregatable') && _.has(field, 'searchable'); + }); } function indexFields(indexPattern) { + let promise = Promise.resolve(); + if (!indexPattern.id) { - return; + return promise; } - if (!indexPattern.fields) { - return indexPattern.refreshFields(); + + if (!indexPattern.fields || !containsFieldCapabilities(indexPattern.fields)) { + promise = indexPattern.refreshFields(); } - initFields(indexPattern); + return promise.then(() => {initFields(indexPattern);}); } function setId(indexPattern, id) { diff --git a/src/ui/public/index_patterns/_mapper.js b/src/ui/public/index_patterns/_mapper.js index 0ee8984327749..b3a0d5b4584e2 100644 --- a/src/ui/public/index_patterns/_mapper.js +++ b/src/ui/public/index_patterns/_mapper.js @@ -1,12 +1,14 @@ import { IndexPatternMissingIndices } from 'ui/errors'; import _ from 'lodash'; import moment from 'moment'; +import EnhanceFieldsWithCapabilitiesProvider from 'ui/index_patterns/_enhance_fields_with_capabilities'; import IndexPatternsTransformMappingIntoFieldsProvider from 'ui/index_patterns/_transform_mapping_into_fields'; import IndexPatternsIntervalsProvider from 'ui/index_patterns/_intervals'; import IndexPatternsPatternToWildcardProvider from 'ui/index_patterns/_pattern_to_wildcard'; import IndexPatternsLocalCacheProvider from 'ui/index_patterns/_local_cache'; export default function MapperService(Private, Promise, es, config, kbnIndex) { + let enhanceFieldsWithCapabilities = Private(EnhanceFieldsWithCapabilitiesProvider); let transformMappingIntoFields = Private(IndexPatternsTransformMappingIntoFieldsProvider); let intervals = Private(IndexPatternsIntervalsProvider); let patternToWildcard = Private(IndexPatternsPatternToWildcardProvider); @@ -49,16 +51,17 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) { }); } - let promise = Promise.resolve(id); + let indexList = id; + let promise = Promise.resolve(); if (indexPattern.intervalName) { promise = self.getIndicesForIndexPattern(indexPattern) .then(function (existing) { if (existing.matches.length === 0) throw new IndexPatternMissingIndices(); - return existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent + indexList = existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent }); } - return promise.then(function (indexList) { + return promise.then(function () { return es.indices.getFieldMapping({ index: indexList, fields: '*', @@ -69,6 +72,7 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) { }) .catch(handleMissingIndexPattern) .then(transformMappingIntoFields) + .then(fields => enhanceFieldsWithCapabilities(fields, indexList)) .then(function (fields) { fieldCache.set(id, fields); return fieldCache.get(id); diff --git a/test/unit/api/ingest/_field_capabilities.js b/test/unit/api/ingest/_field_capabilities.js new file mode 100644 index 0000000000000..e4be4a24b4930 --- /dev/null +++ b/test/unit/api/ingest/_field_capabilities.js @@ -0,0 +1,87 @@ +define(function (require) { + var Promise = require('bluebird'); + var _ = require('intern/dojo/node!lodash'); + var expect = require('intern/dojo/node!expect.js'); + + return function (bdd, scenarioManager, request) { + bdd.describe('field_capabilities API', function postIngest() { + + bdd.before(function () { + return scenarioManager.client.create({ + index: 'foo-1', + type: 'bar', + id: '1', + body: { + foo: 'bar' + } + }) + .then(function () { + return scenarioManager.client.create({ + index: 'foo-2', + type: 'bar', + id: '2', + body: { + baz: 'bar' + } + }); + }) + .then(function () { + return scenarioManager.client.indices.refresh({ + index: ['foo-1', 'foo-2'] + }); + }); + }); + + bdd.after(function () { + return scenarioManager.reload('emptyKibana') + .then(function () { + scenarioManager.client.indices.delete({ + index: 'foo*' + }); + }); + }); + + bdd.it('should return searchable/aggregatable flags for fields in the indices specified', function () { + return request.get('/kibana/foo-1/field_capabilities') + .expect(200) + .then(function (response) { + var fields = response.body.fields; + expect(fields.foo).to.eql({searchable: true, aggregatable: false}); + expect(fields['foo.keyword']).to.eql({searchable: true, aggregatable: true}); + expect(fields).to.not.have.property('baz'); + }); + }); + + bdd.it('should accept wildcards in the index name', function () { + return request.get('/kibana/foo-*/field_capabilities') + .expect(200) + .then(function (response) { + var fields = response.body.fields; + expect(fields.foo).to.eql({searchable: true, aggregatable: false}); + expect(fields.baz).to.eql({searchable: true, aggregatable: false}); + }); + }); + + bdd.it('should accept comma delimited lists of indices', function () { + return request.get('/kibana/foo-1,foo-2/field_capabilities') + .expect(200) + .then(function (response) { + var fields = response.body.fields; + expect(fields.foo).to.eql({searchable: true, aggregatable: false}); + expect(fields.baz).to.eql({searchable: true, aggregatable: false}); + }); + }); + + bdd.it('should return 404 if a pattern matches no indices', function () { + return request.post('/kibana/doesnotexist-*/field_capabilities') + .expect(404); + }); + + bdd.it('should return 404 if a concrete index does not exist', function () { + return request.post('/kibana/concrete/field_capabilities') + .expect(404); + }); + + }); + }; +}); diff --git a/test/unit/api/ingest/index.js b/test/unit/api/ingest/index.js index d83140c7dc9e6..f8b548f75db81 100644 --- a/test/unit/api/ingest/index.js +++ b/test/unit/api/ingest/index.js @@ -12,6 +12,7 @@ define(function (require) { var simulate = require('./_simulate'); var processors = require('./_processors'); var processorTypes = require('./processors/index'); + var fieldCapabilities = require('./_field_capabilities'); bdd.describe('ingest API', function () { var scenarioManager = new ScenarioManager(url.format(serverConfig.servers.elasticsearch)); @@ -31,5 +32,6 @@ define(function (require) { simulate(bdd, scenarioManager, request); processors(bdd, scenarioManager, request); processorTypes(bdd, scenarioManager, request); + fieldCapabilities(bdd, scenarioManager, request); }); });