Skip to content

Commit

Permalink
Merge pull request #8462 from elastic/jasper/backport/8421/5.0
Browse files Browse the repository at this point in the history
[backport] PR #8421 to 5.0
  • Loading branch information
Matt Bargar authored Sep 23, 2016
2 parents 64ef508 + 1c53b0d commit 49e858d
Show file tree
Hide file tree
Showing 11 changed files with 205 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ uiModules.get('apps/management')
{ title: 'name' },
{ title: 'type' },
{ title: 'format' },
{ title: 'searchable', info: 'These fields can be used in the filter bar' },
{ title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' },
{ title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' },
{ title: 'indexed', info: 'Fields that are not indexed are unavailable for search' },
{ title: 'controls', sortable: false }
];

Expand Down Expand Up @@ -54,12 +55,16 @@ uiModules.get('apps/management')
},
_.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']),
{
markup: field.analyzed ? yesTemplate : noTemplate,
value: field.analyzed
markup: field.searchable ? yesTemplate : noTemplate,
value: field.searchable
},
{
markup: field.aggregatable ? yesTemplate : noTemplate,
value: field.aggregatable
},
{
markup: field.indexed ? yesTemplate : noTemplate,
value: field.indexed
markup: field.analyzed ? yesTemplate : noTemplate,
value: field.analyzed
},
{
markup: controlsHtml,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ uiModules
}

function getIndexedFields(param) {
let fields = $scope.agg.vis.indexPattern.fields.raw;
let fields = _.filter($scope.agg.vis.indexPattern.fields.raw, 'aggregatable');
const fieldTypes = param.filterFieldTypes;

if (fieldTypes) {
Expand Down
2 changes: 2 additions & 0 deletions src/core_plugins/kibana/server/routes/api/ingest/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@ import { registerDelete } from './register_delete';
import { registerProcessors } from './register_processors';
import { registerSimulate } from './register_simulate';
import { registerData } from './register_data';
import { registerFieldCapabilities } from './register_field_capabilities';

export default function (server) {
registerPost(server);
registerDelete(server);
registerProcessors(server);
registerSimulate(server);
registerData(server);
registerFieldCapabilities(server);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import _ from 'lodash';
import handleESError from '../../../lib/handle_es_error';

export function registerFieldCapabilities(server) {
server.route({
path: '/api/kibana/{indices}/field_capabilities',
method: ['GET'],
handler: function (req, reply) {
const callWithRequest = server.plugins.elasticsearch.callWithRequest;
const indices = req.params.indices || '';

return callWithRequest(req, 'fieldStats', {
fields: '*',
level: 'cluster',
index: indices,
allowNoIndices: false
})
.catch((error) => {
reply(handleESError(error));
})
.then((res) => {
const fields = _.get(res, 'indices._all.fields', {});
const fieldsFilteredValues = _.mapValues(fields, (value) => {
return _.pick(value, ['searchable', 'aggregatable']);
});

reply({fields: fieldsFilteredValues});
});
}
});
}
37 changes: 29 additions & 8 deletions src/ui/public/index_patterns/__tests__/_index_pattern.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ describe('index pattern', function () {
sinon.stub(mapper, 'getFieldsForIndexPattern', function () {
return Promise.resolve(_.filter(mockLogstashFields, { scripted: false }));
});
sinon.stub(mapper, 'clearCache', function () {
return Promise.resolve();
});

// stub mappingSetup
mappingSetup = Private(UtilsMappingSetupProvider);
Expand Down Expand Up @@ -151,15 +154,34 @@ describe('index pattern', function () {
const indexPatternId = 'test-pattern';
let indexPattern;
let fieldLength;
let truncatedFields;
let customFields;

beforeEach(function () {
fieldLength = mockLogstashFields.length;
truncatedFields = mockLogstashFields.slice(3);
customFields = [{
analyzed: true,
count: 30,
filterable: true,
indexed: true,
name: 'foo',
scripted: false,
sortable: true,
type: 'number',
aggregatable: true,
searchable: false
},
{
name: 'script number',
type: 'number',
scripted: true,
script: '1234',
lang: 'expression'
}];

return create(indexPatternId, {
_source: {
customFormats: '{}',
fields: JSON.stringify(truncatedFields)
fields: JSON.stringify(customFields)
}
}).then(function (pattern) {
indexPattern = pattern;
Expand All @@ -168,8 +190,8 @@ describe('index pattern', function () {

it('should fetch fields from the doc source', function () {
// ensure that we don't have all the fields
expect(truncatedFields.length).to.not.equal(mockLogstashFields.length);
expect(indexPattern.fields).to.have.length(truncatedFields.length);
expect(customFields.length).to.not.equal(mockLogstashFields.length);
expect(indexPattern.fields).to.have.length(customFields.length);

// ensure that all fields will be included in the returned docSource
setDocsourcePayload(docSourceResponse(indexPatternId));
Expand Down Expand Up @@ -201,9 +223,8 @@ describe('index pattern', function () {
// called to append scripted fields to the response from mapper.getFieldsForIndexPattern
expect(scriptedFieldsSpy.callCount).to.equal(1);

const scripted = _.where(mockLogstashFields, { scripted: true });
const expected = _.filter(indexPattern.fields, { scripted: true });
expect(_.pluck(expected, 'name')).to.eql(_.pluck(scripted, 'name'));
const expected = _.filter(indexPattern.fields, {scripted: true});
expect(_.pluck(expected, 'name')).to.eql(['script number']);
});
});
});
Expand Down
16 changes: 16 additions & 0 deletions src/ui/public/index_patterns/_enhance_fields_with_capabilities.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import chrome from 'ui/chrome';
import _ from 'lodash';

export default function ($http) {

return function (fields, indices) {
return $http.get(chrome.addBasePath(`/api/kibana/${indices}/field_capabilities`))
.then((res) => {
const stats = _.get(res, 'data.fields', {});

return _.map(fields, (field) => {
return _.assign(field, stats[field.name]);
});
});
};
}
4 changes: 4 additions & 0 deletions src/ui/public/index_patterns/_field.js
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ export default function FieldObjectProvider(Private, shortDotsFilter, $rootScope
obj.fact('analyzed', !!spec.analyzed);
obj.fact('doc_values', !!spec.doc_values);

// stats
obj.fact('searchable', !!spec.searchable);
obj.fact('aggregatable', !!spec.aggregatable);

// usage flags, read-only and won't be saved
obj.comp('format', format);
obj.comp('sortable', sortable);
Expand Down
21 changes: 16 additions & 5 deletions src/ui/public/index_patterns/_index_pattern.js
Original file line number Diff line number Diff line change
Expand Up @@ -82,23 +82,34 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex,
// give index pattern all of the values in _source
_.assign(indexPattern, response._source);

indexFields(indexPattern);
const promise = indexFields(indexPattern);

// any time index pattern in ES is updated, update index pattern object
docSources
.get(indexPattern)
.onUpdate()
.then(response => updateFromElasticSearch(indexPattern, response), notify.fatal);

return promise;
}

function containsFieldCapabilities(fields) {
return _.any(fields, (field) => {
return _.has(field, 'aggregatable') && _.has(field, 'searchable');
});
}

function indexFields(indexPattern) {
let promise = Promise.resolve();

if (!indexPattern.id) {
return;
return promise;
}
if (!indexPattern.fields) {
return indexPattern.refreshFields();

if (!indexPattern.fields || !containsFieldCapabilities(indexPattern.fields)) {
promise = indexPattern.refreshFields();
}
initFields(indexPattern);
return promise.then(() => {initFields(indexPattern);});
}

function setId(indexPattern, id) {
Expand Down
10 changes: 7 additions & 3 deletions src/ui/public/index_patterns/_mapper.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import { IndexPatternMissingIndices } from 'ui/errors';
import _ from 'lodash';
import moment from 'moment';
import EnhanceFieldsWithCapabilitiesProvider from 'ui/index_patterns/_enhance_fields_with_capabilities';
import IndexPatternsTransformMappingIntoFieldsProvider from 'ui/index_patterns/_transform_mapping_into_fields';
import IndexPatternsIntervalsProvider from 'ui/index_patterns/_intervals';
import IndexPatternsPatternToWildcardProvider from 'ui/index_patterns/_pattern_to_wildcard';
import IndexPatternsLocalCacheProvider from 'ui/index_patterns/_local_cache';
export default function MapperService(Private, Promise, es, config, kbnIndex) {

let enhanceFieldsWithCapabilities = Private(EnhanceFieldsWithCapabilitiesProvider);
let transformMappingIntoFields = Private(IndexPatternsTransformMappingIntoFieldsProvider);
let intervals = Private(IndexPatternsIntervalsProvider);
let patternToWildcard = Private(IndexPatternsPatternToWildcardProvider);
Expand Down Expand Up @@ -49,16 +51,17 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) {
});
}

let promise = Promise.resolve(id);
let indexList = id;
let promise = Promise.resolve();
if (indexPattern.intervalName) {
promise = self.getIndicesForIndexPattern(indexPattern)
.then(function (existing) {
if (existing.matches.length === 0) throw new IndexPatternMissingIndices();
return existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent
indexList = existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent
});
}

return promise.then(function (indexList) {
return promise.then(function () {
return es.indices.getFieldMapping({
index: indexList,
fields: '*',
Expand All @@ -69,6 +72,7 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) {
})
.catch(handleMissingIndexPattern)
.then(transformMappingIntoFields)
.then(fields => enhanceFieldsWithCapabilities(fields, indexList))
.then(function (fields) {
fieldCache.set(id, fields);
return fieldCache.get(id);
Expand Down
87 changes: 87 additions & 0 deletions test/unit/api/ingest/_field_capabilities.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
define(function (require) {
var Promise = require('bluebird');
var _ = require('intern/dojo/node!lodash');
var expect = require('intern/dojo/node!expect.js');

return function (bdd, scenarioManager, request) {
bdd.describe('field_capabilities API', function postIngest() {

bdd.before(function () {
return scenarioManager.client.create({
index: 'foo-1',
type: 'bar',
id: '1',
body: {
foo: 'bar'
}
})
.then(function () {
return scenarioManager.client.create({
index: 'foo-2',
type: 'bar',
id: '2',
body: {
baz: 'bar'
}
});
})
.then(function () {
return scenarioManager.client.indices.refresh({
index: ['foo-1', 'foo-2']
});
});
});

bdd.after(function () {
return scenarioManager.reload('emptyKibana')
.then(function () {
scenarioManager.client.indices.delete({
index: 'foo*'
});
});
});

bdd.it('should return searchable/aggregatable flags for fields in the indices specified', function () {
return request.get('/kibana/foo-1/field_capabilities')
.expect(200)
.then(function (response) {
var fields = response.body.fields;
expect(fields.foo).to.eql({searchable: true, aggregatable: false});
expect(fields['foo.keyword']).to.eql({searchable: true, aggregatable: true});
expect(fields).to.not.have.property('baz');
});
});

bdd.it('should accept wildcards in the index name', function () {
return request.get('/kibana/foo-*/field_capabilities')
.expect(200)
.then(function (response) {
var fields = response.body.fields;
expect(fields.foo).to.eql({searchable: true, aggregatable: false});
expect(fields.baz).to.eql({searchable: true, aggregatable: false});
});
});

bdd.it('should accept comma delimited lists of indices', function () {
return request.get('/kibana/foo-1,foo-2/field_capabilities')
.expect(200)
.then(function (response) {
var fields = response.body.fields;
expect(fields.foo).to.eql({searchable: true, aggregatable: false});
expect(fields.baz).to.eql({searchable: true, aggregatable: false});
});
});

bdd.it('should return 404 if a pattern matches no indices', function () {
return request.post('/kibana/doesnotexist-*/field_capabilities')
.expect(404);
});

bdd.it('should return 404 if a concrete index does not exist', function () {
return request.post('/kibana/concrete/field_capabilities')
.expect(404);
});

});
};
});
2 changes: 2 additions & 0 deletions test/unit/api/ingest/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ define(function (require) {
var simulate = require('./_simulate');
var processors = require('./_processors');
var processorTypes = require('./processors/index');
var fieldCapabilities = require('./_field_capabilities');

bdd.describe('ingest API', function () {
var scenarioManager = new ScenarioManager(url.format(serverConfig.servers.elasticsearch));
Expand All @@ -31,5 +32,6 @@ define(function (require) {
simulate(bdd, scenarioManager, request);
processors(bdd, scenarioManager, request);
processorTypes(bdd, scenarioManager, request);
fieldCapabilities(bdd, scenarioManager, request);
});
});

0 comments on commit 49e858d

Please sign in to comment.