diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx index fca958a39b086..0871ef4749496 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx @@ -116,27 +116,6 @@ describe('IndexPattern Field Item', () => { ); }); - it('should request field stats without a time field, if the index pattern has none', async () => { - indexPattern.timeFieldName = undefined; - core.http.post.mockImplementationOnce(() => { - return Promise.resolve({}); - }); - const wrapper = mountWithIntl(); - - await act(async () => { - clickField(wrapper, 'bytes'); - }); - - expect(core.http.post).toHaveBeenCalledWith( - '/api/lens/index_stats/my-fake-index-pattern/field', - expect.anything() - ); - // Function argument types not detected correctly (https://github.com/microsoft/TypeScript/issues/26591) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const { body } = (core.http.post.mock.calls[0] as any)[1]; - expect(JSON.parse(body)).not.toHaveProperty('timeFieldName'); - }); - it('should request field stats every time the button is clicked', async () => { let resolveFunction: (arg: unknown) => void; @@ -150,31 +129,21 @@ describe('IndexPattern Field Item', () => { clickField(wrapper, 'bytes'); - expect(core.http.post).toHaveBeenCalledWith( - `/api/lens/index_stats/my-fake-index-pattern/field`, - { - body: JSON.stringify({ - dslQuery: { - bool: { - must: [{ match_all: {} }], - filter: [], - should: [], - must_not: [], - }, + expect(core.http.post).toHaveBeenCalledWith(`/api/lens/index_stats/1/field`, { + body: JSON.stringify({ + dslQuery: { + bool: { + must: [{ match_all: {} }], + filter: [], + should: [], + must_not: [], }, - fromDate: 'now-7d', - toDate: 'now', - timeFieldName: 'timestamp', - field: { - name: 'bytes', - displayName: 'bytesLabel', - type: 'number', - aggregatable: true, - searchable: true, - }, - }), - } - ); + }, + fromDate: 'now-7d', + toDate: 'now', + fieldName: 'bytes', + }), + }); expect(wrapper.find(EuiPopover).prop('isOpen')).toEqual(true); @@ -227,40 +196,30 @@ describe('IndexPattern Field Item', () => { clickField(wrapper, 'bytes'); expect(core.http.post).toHaveBeenCalledTimes(2); - expect(core.http.post).toHaveBeenLastCalledWith( - `/api/lens/index_stats/my-fake-index-pattern/field`, - { - body: JSON.stringify({ - dslQuery: { - bool: { - must: [], - filter: [ - { - bool: { - should: [{ match_phrase: { 'geo.src': 'US' } }], - minimum_should_match: 1, - }, - }, - { - match: { phrase: { 'geo.dest': 'US' } }, + expect(core.http.post).toHaveBeenLastCalledWith(`/api/lens/index_stats/1/field`, { + body: JSON.stringify({ + dslQuery: { + bool: { + must: [], + filter: [ + { + bool: { + should: [{ match_phrase: { 'geo.src': 'US' } }], + minimum_should_match: 1, }, - ], - should: [], - must_not: [], - }, + }, + { + match: { phrase: { 'geo.dest': 'US' } }, + }, + ], + should: [], + must_not: [], }, - fromDate: 'now-14d', - toDate: 'now-7d', - timeFieldName: 'timestamp', - field: { - name: 'bytes', - displayName: 'bytesLabel', - type: 'number', - aggregatable: true, - searchable: true, - }, - }), - } - ); + }, + fromDate: 'now-14d', + toDate: 'now-7d', + fieldName: 'bytes', + }), + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx index e0198d6d7903e..e5d46b4a7a073 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx @@ -129,7 +129,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) { setState((s) => ({ ...s, isLoading: true })); core.http - .post(`/api/lens/index_stats/${indexPattern.title}/field`, { + .post(`/api/lens/index_stats/${indexPattern.id}/field`, { body: JSON.stringify({ dslQuery: esQuery.buildEsQuery( indexPattern as IIndexPattern, @@ -139,8 +139,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) { ), fromDate: dateRange.fromDate, toDate: dateRange.toDate, - timeFieldName: indexPattern.timeFieldName, - field, + fieldName: field.name, }), }) .then((results: FieldStatsResponse) => { diff --git a/x-pack/plugins/lens/server/routes/existing_fields.test.ts b/x-pack/plugins/lens/server/routes/existing_fields.test.ts index c6364eca0ff49..3f3e94099f666 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.test.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.test.ts @@ -61,6 +61,20 @@ describe('existingFields', () => { expect(result).toEqual(['bar']); }); + it('supports runtime fields', () => { + const result = existingFields( + [searchResults({ runtime_foo: ['scriptvalue'] })], + [ + field({ + name: 'runtime_foo', + runtimeField: { type: 'long', script: { source: '2+2' } }, + }), + ] + ); + + expect(result).toEqual(['runtime_foo']); + }); + it('supports meta fields', () => { const result = existingFields( [{ _mymeta: 'abc', ...searchResults({ bar: ['scriptvalue'] }) }], @@ -78,6 +92,11 @@ describe('buildFieldList', () => { typeMeta: 'typemeta', fields: [ { name: 'foo', scripted: true, lang: 'painless', script: '2+2' }, + { + name: 'runtime_foo', + isMapped: false, + runtimeField: { type: 'long', script: { source: '2+2' } }, + }, { name: 'bar' }, { name: '@bar' }, { name: 'baz' }, @@ -95,6 +114,14 @@ describe('buildFieldList', () => { }); }); + it('supports runtime fields', () => { + const fields = buildFieldList((indexPattern as unknown) as IndexPattern, []); + expect(fields.find((f) => f.runtimeField)).toMatchObject({ + name: 'runtime_foo', + runtimeField: { type: 'long', script: { source: '2+2' } }, + }); + }); + it('supports meta fields', () => { const fields = buildFieldList((indexPattern as unknown) as IndexPattern, ['_mymeta']); expect(fields.find((f) => f.isMeta)).toMatchObject({ diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index e76abf4598efa..11db9360749ea 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -10,7 +10,7 @@ import { errors } from '@elastic/elasticsearch'; import { schema } from '@kbn/config-schema'; import { RequestHandlerContext, ElasticsearchClient } from 'src/core/server'; import { CoreSetup, Logger } from 'src/core/server'; -import { IndexPattern, IndexPatternsService } from 'src/plugins/data/common'; +import { IndexPattern, IndexPatternsService, RuntimeField } from 'src/plugins/data/common'; import { BASE_API_URL } from '../../common'; import { UI_SETTINGS } from '../../../../../src/plugins/data/server'; import { PluginStartContract } from '../plugin'; @@ -30,6 +30,7 @@ export interface Field { isMeta: boolean; lang?: string; script?: string; + runtimeField?: RuntimeField; } export async function existingFieldsRoute(setup: CoreSetup, logger: Logger) { @@ -138,6 +139,7 @@ export function buildFieldList(indexPattern: IndexPattern, metaFields: string[]) // id is a special case - it doesn't show up in the meta field list, // but as it's not part of source, it has to be handled separately. isMeta: metaFields.includes(field.name) || field.name === '_id', + runtimeField: !field.isMapped ? field.runtimeField : undefined, }; }); } @@ -181,6 +183,7 @@ async function fetchIndexPatternStats({ }; const scriptedFields = fields.filter((f) => f.isScript); + const runtimeFields = fields.filter((f) => f.runtimeField); const { body: result } = await client.search({ index, body: { @@ -189,6 +192,11 @@ async function fetchIndexPatternStats({ sort: timeFieldName && fromDate && toDate ? [{ [timeFieldName]: 'desc' }] : [], fields: ['*'], _source: false, + runtime_mappings: runtimeFields.reduce((acc, field) => { + if (!field.runtimeField) return acc; + acc[field.name] = field.runtimeField; + return acc; + }, {} as Record), script_fields: scriptedFields.reduce((acc, field) => { acc[field.name] = { script: { diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index 7fd884755d86d..9094e5442dc51 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -11,6 +11,7 @@ import DateMath from '@elastic/datemath'; import { schema } from '@kbn/config-schema'; import { CoreSetup } from 'src/core/server'; import { IFieldType } from 'src/plugins/data/common'; +import { SavedObjectNotFound } from '../../../../../src/plugins/kibana_utils/common'; import { ESSearchResponse } from '../../../../typings/elasticsearch'; import { FieldStatsResponse, BASE_API_URL } from '../../common'; import { PluginStartContract } from '../plugin'; @@ -21,28 +22,17 @@ export async function initFieldsRoute(setup: CoreSetup) { const router = setup.http.createRouter(); router.post( { - path: `${BASE_API_URL}/index_stats/{indexPatternTitle}/field`, + path: `${BASE_API_URL}/index_stats/{indexPatternId}/field`, validate: { params: schema.object({ - indexPatternTitle: schema.string(), + indexPatternId: schema.string(), }), body: schema.object( { dslQuery: schema.object({}, { unknowns: 'allow' }), fromDate: schema.string(), toDate: schema.string(), - timeFieldName: schema.maybe(schema.string()), - field: schema.object( - { - name: schema.string(), - type: schema.string(), - esTypes: schema.maybe(schema.arrayOf(schema.string())), - scripted: schema.maybe(schema.boolean()), - lang: schema.maybe(schema.string()), - script: schema.maybe(schema.string()), - }, - { unknowns: 'allow' } - ), + fieldName: schema.string(), }, { unknowns: 'allow' } ), @@ -50,9 +40,26 @@ export async function initFieldsRoute(setup: CoreSetup) { }, async (context, req, res) => { const requestClient = context.core.elasticsearch.client.asCurrentUser; - const { fromDate, toDate, timeFieldName, field, dslQuery } = req.body; + const { fromDate, toDate, fieldName, dslQuery } = req.body; + + const [{ savedObjects, elasticsearch }, { data }] = await setup.getStartServices(); + const savedObjectsClient = savedObjects.getScopedClient(req); + const esClient = elasticsearch.client.asScoped(req).asCurrentUser; + const indexPatternsService = await data.indexPatterns.indexPatternsServiceFactory( + savedObjectsClient, + esClient + ); try { + const indexPattern = await indexPatternsService.get(req.params.indexPatternId); + + const timeFieldName = indexPattern.timeFieldName; + const field = indexPattern.fields.find((f) => f.name === fieldName); + + if (!field) { + throw new Error(`Field {fieldName} not found in index pattern ${indexPattern.title}`); + } + const filter = timeFieldName ? [ { @@ -75,11 +82,12 @@ export async function initFieldsRoute(setup: CoreSetup) { const search = async (aggs: unknown) => { const { body: result } = await requestClient.search({ - index: req.params.indexPatternTitle, + index: indexPattern.title, track_total_hits: true, body: { query, aggs, + runtime_mappings: field.runtimeField ? { [fieldName]: field.runtimeField } : {}, }, size: 0, }); @@ -104,6 +112,9 @@ export async function initFieldsRoute(setup: CoreSetup) { body: await getStringSamples(search, field), }); } catch (e) { + if (e instanceof SavedObjectNotFound) { + return res.notFound(); + } if (e instanceof errors.ResponseError && e.statusCode === 404) { return res.notFound(); } diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts index ac4ebb4e5b02c..94960b9859121 100644 --- a/x-pack/test/api_integration/apis/lens/field_stats.ts +++ b/x-pack/test/api_integration/apis/lens/field_stats.ts @@ -22,29 +22,29 @@ export default ({ getService }: FtrProviderContext) => { describe('index stats apis', () => { before(async () => { await esArchiver.loadIfNeeded('logstash_functional'); - await esArchiver.loadIfNeeded('visualize/default'); - await esArchiver.loadIfNeeded('pre_calculated_histogram'); }); after(async () => { await esArchiver.unload('logstash_functional'); - await esArchiver.unload('visualize/default'); - await esArchiver.unload('pre_calculated_histogram'); }); describe('field distribution', () => { + before(async () => { + await esArchiver.loadIfNeeded('visualize/default'); + }); + after(async () => { + await esArchiver.unload('visualize/default'); + }); + it('should return a 404 for missing index patterns', async () => { await supertest - .post('/api/lens/index_stats/logstash/field') + .post('/api/lens/index_stats/123/field') .set(COMMON_HEADERS) .send({ dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, timeFieldName: '@timestamp', - field: { - name: 'bytes', - type: 'number', - }, + fieldName: 'bytes', }) .expect(404); }); @@ -57,10 +57,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - field: { - name: 'bytes', - type: 'number', - }, + fieldName: 'bytes', }) .expect(200); @@ -75,11 +72,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'bytes', - type: 'number', - }, + fieldName: 'bytes', }) .expect(200); @@ -186,11 +179,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: '@timestamp', - type: 'date', - }, + fieldName: '@timestamp', }) .expect(200); @@ -223,11 +212,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'geo.src', - type: 'string', - }, + fieldName: 'geo.src', }) .expect(200); @@ -290,11 +275,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'ip', - type: 'ip', - }, + fieldName: 'ip', }) .expect(200); @@ -349,6 +330,113 @@ export default ({ getService }: FtrProviderContext) => { }); }); + it('should return histograms for scripted date fields', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + fieldName: 'scripted_date', + }) + .expect(200); + + expect(body).to.eql({ + histogram: { + buckets: [ + { + count: 4634, + key: 0, + }, + ], + }, + totalDocuments: 4634, + }); + }); + + it('should return top values for scripted string fields', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + fieldName: 'scripted_string', + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4634, + sampledDocuments: 4634, + sampledValues: 4634, + topValues: { + buckets: [ + { + count: 4634, + key: 'hello', + }, + ], + }, + }); + }); + + it('should return top values for index pattern runtime string fields', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + fieldName: 'runtime_string_field', + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4634, + sampledDocuments: 4634, + sampledValues: 4634, + topValues: { + buckets: [ + { + count: 4634, + key: 'hello world!', + }, + ], + }, + }); + }); + + it('should apply filters and queries', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { + bool: { + filter: [{ match: { 'geo.src': 'US' } }], + }, + }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + fieldName: 'bytes', + }) + .expect(200); + + expect(body.totalDocuments).to.eql(425); + }); + }); + + describe('histogram', () => { + before(async () => { + await esArchiver.loadIfNeeded('pre_calculated_histogram'); + }); + after(async () => { + await esArchiver.unload('pre_calculated_histogram'); + }); + it('should return an auto histogram for precalculated histograms', async () => { const { body } = await supertest .post('/api/lens/index_stats/histogram-test/field') @@ -357,10 +445,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - field: { - name: 'histogram-content', - type: 'histogram', - }, + fieldName: 'histogram-content', }) .expect(200); @@ -428,10 +513,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match: { 'histogram-title': 'single value' } }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - field: { - name: 'histogram-content', - type: 'histogram', - }, + fieldName: 'histogram-content', }) .expect(200); @@ -443,95 +525,6 @@ export default ({ getService }: FtrProviderContext) => { topValues: { buckets: [] }, }); }); - - it('should return histograms for scripted date fields', async () => { - const { body } = await supertest - .post('/api/lens/index_stats/logstash-2015.09.22/field') - .set(COMMON_HEADERS) - .send({ - dslQuery: { match_all: {} }, - fromDate: TEST_START_TIME, - toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'scripted date', - type: 'date', - scripted: true, - script: '1234', - lang: 'painless', - }, - }) - .expect(200); - - expect(body).to.eql({ - histogram: { - buckets: [ - { - count: 4634, - key: 0, - }, - ], - }, - totalDocuments: 4634, - }); - }); - - it('should return top values for scripted string fields', async () => { - const { body } = await supertest - .post('/api/lens/index_stats/logstash-2015.09.22/field') - .set(COMMON_HEADERS) - .send({ - dslQuery: { match_all: {} }, - fromDate: TEST_START_TIME, - toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'scripted string', - type: 'string', - scripted: true, - script: 'return "hello"', - lang: 'painless', - }, - }) - .expect(200); - - expect(body).to.eql({ - totalDocuments: 4634, - sampledDocuments: 4634, - sampledValues: 4634, - topValues: { - buckets: [ - { - count: 4634, - key: 'hello', - }, - ], - }, - }); - }); - - it('should apply filters and queries', async () => { - const { body } = await supertest - .post('/api/lens/index_stats/logstash-2015.09.22/field') - .set(COMMON_HEADERS) - .send({ - dslQuery: { - bool: { - filter: [{ match: { 'geo.src': 'US' } }], - }, - }, - fromDate: TEST_START_TIME, - toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'bytes', - type: 'number', - }, - }) - .expect(200); - - expect(body.totalDocuments).to.eql(425); - }); }); }); }; diff --git a/x-pack/test/functional/es_archives/visualize/default/data.json b/x-pack/test/functional/es_archives/visualize/default/data.json index 26b033e28b4da..7d0ad0c25f96d 100644 --- a/x-pack/test/functional/es_archives/visualize/default/data.json +++ b/x-pack/test/functional/es_archives/visualize/default/data.json @@ -145,6 +145,31 @@ } } +{ + "type": "doc", + "value": { + "index": ".kibana", + "type": "doc", + "id": "index-pattern:logstash-2015.09.22", + "index": ".kibana_1", + "source": { + "index-pattern": { + "timeFieldName": "@timestamp", + "title": "logstash-2015.09.22", + "fields":"[{\"name\":\"scripted_date\",\"type\":\"date\",\"count\":0,\"scripted\":true,\"script\":\"1234\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"scripted_string\",\"type\":\"string\",\"count\":0,\"scripted\":true,\"script\":\"return 'hello'\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]", + "runtimeFieldMap":"{\"runtime_string_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit('hello world!')\"}}}" + }, + "migrationVersion": { + "index-pattern": "7.11.0" + }, + "references": [ + ], + "type": "index-pattern", + "updated_at": "2018-12-21T00:43:07.096Z" + } + } +} + { "type": "doc", "value": {