From 888d6e65b3dc9c518d5cd1b4ec8bc5aa0e8232aa Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 8 Feb 2021 13:50:50 +0100 Subject: [PATCH 1/5] fix runtime existence --- .../field_item.test.tsx | 101 +++++++----------- .../indexpattern_datasource/field_item.tsx | 4 +- .../server/routes/existing_fields.test.ts | 27 +++++ .../lens/server/routes/existing_fields.ts | 10 +- .../plugins/lens/server/routes/field_stats.ts | 38 ++++--- 5 files changed, 101 insertions(+), 79 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx index fca958a39b086..93b57dc32f452 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx @@ -127,10 +127,7 @@ describe('IndexPattern Field Item', () => { clickField(wrapper, 'bytes'); }); - expect(core.http.post).toHaveBeenCalledWith( - '/api/lens/index_stats/my-fake-index-pattern/field', - expect.anything() - ); + expect(core.http.post).toHaveBeenCalledWith('/api/lens/index_stats/1/field', expect.anything()); // Function argument types not detected correctly (https://github.com/microsoft/TypeScript/issues/26591) // eslint-disable-next-line @typescript-eslint/no-explicit-any const { body } = (core.http.post.mock.calls[0] as any)[1]; @@ -150,31 +147,22 @@ describe('IndexPattern Field Item', () => { clickField(wrapper, 'bytes'); - expect(core.http.post).toHaveBeenCalledWith( - `/api/lens/index_stats/my-fake-index-pattern/field`, - { - body: JSON.stringify({ - dslQuery: { - bool: { - must: [{ match_all: {} }], - filter: [], - should: [], - must_not: [], - }, - }, - fromDate: 'now-7d', - toDate: 'now', - timeFieldName: 'timestamp', - field: { - name: 'bytes', - displayName: 'bytesLabel', - type: 'number', - aggregatable: true, - searchable: true, + expect(core.http.post).toHaveBeenCalledWith(`/api/lens/index_stats/1/field`, { + body: JSON.stringify({ + dslQuery: { + bool: { + must: [{ match_all: {} }], + filter: [], + should: [], + must_not: [], }, - }), - } - ); + }, + fromDate: 'now-7d', + toDate: 'now', + timeFieldName: 'timestamp', + fieldName: 'bytes', + }), + }); expect(wrapper.find(EuiPopover).prop('isOpen')).toEqual(true); @@ -227,40 +215,31 @@ describe('IndexPattern Field Item', () => { clickField(wrapper, 'bytes'); expect(core.http.post).toHaveBeenCalledTimes(2); - expect(core.http.post).toHaveBeenLastCalledWith( - `/api/lens/index_stats/my-fake-index-pattern/field`, - { - body: JSON.stringify({ - dslQuery: { - bool: { - must: [], - filter: [ - { - bool: { - should: [{ match_phrase: { 'geo.src': 'US' } }], - minimum_should_match: 1, - }, + expect(core.http.post).toHaveBeenLastCalledWith(`/api/lens/index_stats/1/field`, { + body: JSON.stringify({ + dslQuery: { + bool: { + must: [], + filter: [ + { + bool: { + should: [{ match_phrase: { 'geo.src': 'US' } }], + minimum_should_match: 1, }, - { - match: { phrase: { 'geo.dest': 'US' } }, - }, - ], - should: [], - must_not: [], - }, - }, - fromDate: 'now-14d', - toDate: 'now-7d', - timeFieldName: 'timestamp', - field: { - name: 'bytes', - displayName: 'bytesLabel', - type: 'number', - aggregatable: true, - searchable: true, + }, + { + match: { phrase: { 'geo.dest': 'US' } }, + }, + ], + should: [], + must_not: [], }, - }), - } - ); + }, + fromDate: 'now-14d', + toDate: 'now-7d', + timeFieldName: 'timestamp', + fieldName: 'bytes', + }), + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx index e0198d6d7903e..b28638f15f9ec 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx @@ -129,7 +129,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) { setState((s) => ({ ...s, isLoading: true })); core.http - .post(`/api/lens/index_stats/${indexPattern.title}/field`, { + .post(`/api/lens/index_stats/${indexPattern.id}/field`, { body: JSON.stringify({ dslQuery: esQuery.buildEsQuery( indexPattern as IIndexPattern, @@ -140,7 +140,7 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) { fromDate: dateRange.fromDate, toDate: dateRange.toDate, timeFieldName: indexPattern.timeFieldName, - field, + fieldName: field.name, }), }) .then((results: FieldStatsResponse) => { diff --git a/x-pack/plugins/lens/server/routes/existing_fields.test.ts b/x-pack/plugins/lens/server/routes/existing_fields.test.ts index c6364eca0ff49..3f3e94099f666 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.test.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.test.ts @@ -61,6 +61,20 @@ describe('existingFields', () => { expect(result).toEqual(['bar']); }); + it('supports runtime fields', () => { + const result = existingFields( + [searchResults({ runtime_foo: ['scriptvalue'] })], + [ + field({ + name: 'runtime_foo', + runtimeField: { type: 'long', script: { source: '2+2' } }, + }), + ] + ); + + expect(result).toEqual(['runtime_foo']); + }); + it('supports meta fields', () => { const result = existingFields( [{ _mymeta: 'abc', ...searchResults({ bar: ['scriptvalue'] }) }], @@ -78,6 +92,11 @@ describe('buildFieldList', () => { typeMeta: 'typemeta', fields: [ { name: 'foo', scripted: true, lang: 'painless', script: '2+2' }, + { + name: 'runtime_foo', + isMapped: false, + runtimeField: { type: 'long', script: { source: '2+2' } }, + }, { name: 'bar' }, { name: '@bar' }, { name: 'baz' }, @@ -95,6 +114,14 @@ describe('buildFieldList', () => { }); }); + it('supports runtime fields', () => { + const fields = buildFieldList((indexPattern as unknown) as IndexPattern, []); + expect(fields.find((f) => f.runtimeField)).toMatchObject({ + name: 'runtime_foo', + runtimeField: { type: 'long', script: { source: '2+2' } }, + }); + }); + it('supports meta fields', () => { const fields = buildFieldList((indexPattern as unknown) as IndexPattern, ['_mymeta']); expect(fields.find((f) => f.isMeta)).toMatchObject({ diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index e76abf4598efa..11db9360749ea 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -10,7 +10,7 @@ import { errors } from '@elastic/elasticsearch'; import { schema } from '@kbn/config-schema'; import { RequestHandlerContext, ElasticsearchClient } from 'src/core/server'; import { CoreSetup, Logger } from 'src/core/server'; -import { IndexPattern, IndexPatternsService } from 'src/plugins/data/common'; +import { IndexPattern, IndexPatternsService, RuntimeField } from 'src/plugins/data/common'; import { BASE_API_URL } from '../../common'; import { UI_SETTINGS } from '../../../../../src/plugins/data/server'; import { PluginStartContract } from '../plugin'; @@ -30,6 +30,7 @@ export interface Field { isMeta: boolean; lang?: string; script?: string; + runtimeField?: RuntimeField; } export async function existingFieldsRoute(setup: CoreSetup, logger: Logger) { @@ -138,6 +139,7 @@ export function buildFieldList(indexPattern: IndexPattern, metaFields: string[]) // id is a special case - it doesn't show up in the meta field list, // but as it's not part of source, it has to be handled separately. isMeta: metaFields.includes(field.name) || field.name === '_id', + runtimeField: !field.isMapped ? field.runtimeField : undefined, }; }); } @@ -181,6 +183,7 @@ async function fetchIndexPatternStats({ }; const scriptedFields = fields.filter((f) => f.isScript); + const runtimeFields = fields.filter((f) => f.runtimeField); const { body: result } = await client.search({ index, body: { @@ -189,6 +192,11 @@ async function fetchIndexPatternStats({ sort: timeFieldName && fromDate && toDate ? [{ [timeFieldName]: 'desc' }] : [], fields: ['*'], _source: false, + runtime_mappings: runtimeFields.reduce((acc, field) => { + if (!field.runtimeField) return acc; + acc[field.name] = field.runtimeField; + return acc; + }, {} as Record), script_fields: scriptedFields.reduce((acc, field) => { acc[field.name] = { script: { diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index e1681a74c2951..8b3607b428209 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -21,10 +21,10 @@ export async function initFieldsRoute(setup: CoreSetup) { const router = setup.http.createRouter(); router.post( { - path: `${BASE_API_URL}/index_stats/{indexPatternTitle}/field`, + path: `${BASE_API_URL}/index_stats/{indexPatternId}/field`, validate: { params: schema.object({ - indexPatternTitle: schema.string(), + indexPatternId: schema.string(), }), body: schema.object( { @@ -32,17 +32,7 @@ export async function initFieldsRoute(setup: CoreSetup) { fromDate: schema.string(), toDate: schema.string(), timeFieldName: schema.maybe(schema.string()), - field: schema.object( - { - name: schema.string(), - type: schema.string(), - esTypes: schema.maybe(schema.arrayOf(schema.string())), - scripted: schema.maybe(schema.boolean()), - lang: schema.maybe(schema.string()), - script: schema.maybe(schema.string()), - }, - { unknowns: 'allow' } - ), + fieldName: schema.string(), }, { unknowns: 'allow' } ), @@ -50,7 +40,23 @@ export async function initFieldsRoute(setup: CoreSetup) { }, async (context, req, res) => { const requestClient = context.core.elasticsearch.client.asCurrentUser; - const { fromDate, toDate, timeFieldName, field, dslQuery } = req.body; + const { fromDate, toDate, timeFieldName, fieldName, dslQuery } = req.body; + + const [{ savedObjects, elasticsearch }, { data }] = await setup.getStartServices(); + const savedObjectsClient = savedObjects.getScopedClient(req); + const esClient = elasticsearch.client.asScoped(req).asCurrentUser; + const indexPatternsService = await data.indexPatterns.indexPatternsServiceFactory( + savedObjectsClient, + esClient + ); + + const indexPattern = await indexPatternsService.get(req.params.indexPatternId); + + const field = indexPattern.fields.find((f) => f.name === fieldName); + + if (!field) { + throw new Error(`Field {fieldName} not found in index pattern ${indexPattern.title}`); + } try { const filter = timeFieldName @@ -75,11 +81,13 @@ export async function initFieldsRoute(setup: CoreSetup) { const search = async (aggs: unknown) => { const { body: result } = await requestClient.search({ - index: req.params.indexPatternTitle, + index: indexPattern.title, track_total_hits: true, body: { query, aggs, + runtime_mappings: + !field.isMapped && field.runtimeField ? { [fieldName]: field.runtimeField } : {}, }, size: 0, }); From e984392b7ae2ec3efe37bcba5af1b46ff492ad57 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 9 Feb 2021 17:24:54 +0100 Subject: [PATCH 2/5] fix tests --- .../field_item.test.tsx | 20 ------- .../indexpattern_datasource/field_item.tsx | 1 - .../plugins/lens/server/routes/field_stats.ts | 20 ++++--- .../api_integration/apis/lens/field_stats.ts | 60 ++++--------------- .../es_archives/visualize/default/data.json | 24 ++++++++ 5 files changed, 46 insertions(+), 79 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx index 93b57dc32f452..0871ef4749496 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx @@ -116,24 +116,6 @@ describe('IndexPattern Field Item', () => { ); }); - it('should request field stats without a time field, if the index pattern has none', async () => { - indexPattern.timeFieldName = undefined; - core.http.post.mockImplementationOnce(() => { - return Promise.resolve({}); - }); - const wrapper = mountWithIntl(); - - await act(async () => { - clickField(wrapper, 'bytes'); - }); - - expect(core.http.post).toHaveBeenCalledWith('/api/lens/index_stats/1/field', expect.anything()); - // Function argument types not detected correctly (https://github.com/microsoft/TypeScript/issues/26591) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const { body } = (core.http.post.mock.calls[0] as any)[1]; - expect(JSON.parse(body)).not.toHaveProperty('timeFieldName'); - }); - it('should request field stats every time the button is clicked', async () => { let resolveFunction: (arg: unknown) => void; @@ -159,7 +141,6 @@ describe('IndexPattern Field Item', () => { }, fromDate: 'now-7d', toDate: 'now', - timeFieldName: 'timestamp', fieldName: 'bytes', }), }); @@ -237,7 +218,6 @@ describe('IndexPattern Field Item', () => { }, fromDate: 'now-14d', toDate: 'now-7d', - timeFieldName: 'timestamp', fieldName: 'bytes', }), }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx index b28638f15f9ec..e5d46b4a7a073 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx @@ -139,7 +139,6 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) { ), fromDate: dateRange.fromDate, toDate: dateRange.toDate, - timeFieldName: indexPattern.timeFieldName, fieldName: field.name, }), }) diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index 8b3607b428209..907a9c45b74f6 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -11,6 +11,7 @@ import DateMath from '@elastic/datemath'; import { schema } from '@kbn/config-schema'; import { CoreSetup } from 'src/core/server'; import { IFieldType } from 'src/plugins/data/common'; +import { SavedObjectNotFound } from '../../../../../src/plugins/kibana_utils/common'; import { ESSearchResponse } from '../../../../typings/elasticsearch'; import { FieldStatsResponse, BASE_API_URL } from '../../common'; import { PluginStartContract } from '../plugin'; @@ -31,7 +32,6 @@ export async function initFieldsRoute(setup: CoreSetup) { dslQuery: schema.object({}, { unknowns: 'allow' }), fromDate: schema.string(), toDate: schema.string(), - timeFieldName: schema.maybe(schema.string()), fieldName: schema.string(), }, { unknowns: 'allow' } @@ -40,7 +40,7 @@ export async function initFieldsRoute(setup: CoreSetup) { }, async (context, req, res) => { const requestClient = context.core.elasticsearch.client.asCurrentUser; - const { fromDate, toDate, timeFieldName, fieldName, dslQuery } = req.body; + const { fromDate, toDate, fieldName, dslQuery } = req.body; const [{ savedObjects, elasticsearch }, { data }] = await setup.getStartServices(); const savedObjectsClient = savedObjects.getScopedClient(req); @@ -50,15 +50,16 @@ export async function initFieldsRoute(setup: CoreSetup) { esClient ); - const indexPattern = await indexPatternsService.get(req.params.indexPatternId); + try { + const indexPattern = await indexPatternsService.get(req.params.indexPatternId); - const field = indexPattern.fields.find((f) => f.name === fieldName); + const timeFieldName = indexPattern.timeFieldName; + const field = indexPattern.fields.find((f) => f.name === fieldName); - if (!field) { - throw new Error(`Field {fieldName} not found in index pattern ${indexPattern.title}`); - } + if (!field) { + throw new Error(`Field {fieldName} not found in index pattern ${indexPattern.title}`); + } - try { const filter = timeFieldName ? [ { @@ -108,6 +109,9 @@ export async function initFieldsRoute(setup: CoreSetup) { body: await getStringSamples(search, field), }); } catch (e) { + if (e instanceof SavedObjectNotFound) { + return res.notFound(); + } if (e instanceof errors.ResponseError && e.statusCode === 404) { return res.notFound(); } diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts index 2cfce5ef31305..80cf5149f0496 100644 --- a/x-pack/test/api_integration/apis/lens/field_stats.ts +++ b/x-pack/test/api_integration/apis/lens/field_stats.ts @@ -32,17 +32,14 @@ export default ({ getService }: FtrProviderContext) => { describe('field distribution', () => { it('should return a 404 for missing index patterns', async () => { await supertest - .post('/api/lens/index_stats/logstash/field') + .post('/api/lens/index_stats/123/field') .set(COMMON_HEADERS) .send({ dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, timeFieldName: '@timestamp', - field: { - name: 'bytes', - type: 'number', - }, + fieldName: 'bytes', }) .expect(404); }); @@ -55,10 +52,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - field: { - name: 'bytes', - type: 'number', - }, + fieldName: 'bytes', }) .expect(200); @@ -73,11 +67,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'bytes', - type: 'number', - }, + fieldName: 'bytes', }) .expect(200); @@ -184,11 +174,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: '@timestamp', - type: 'date', - }, + fieldName: '@timestamp', }) .expect(200); @@ -221,11 +207,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'geo.src', - type: 'string', - }, + fieldName: 'geo.src', }) .expect(200); @@ -288,11 +270,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'ip', - type: 'ip', - }, + fieldName: 'ip', }) .expect(200); @@ -355,14 +333,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'scripted date', - type: 'date', - scripted: true, - script: '1234', - lang: 'painless', - }, + fieldName: 'scripted_date', }) .expect(200); @@ -387,14 +358,7 @@ export default ({ getService }: FtrProviderContext) => { dslQuery: { match_all: {} }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'scripted string', - type: 'string', - scripted: true, - script: 'return "hello"', - lang: 'painless', - }, + fieldName: 'scripted_string', }) .expect(200); @@ -425,11 +389,7 @@ export default ({ getService }: FtrProviderContext) => { }, fromDate: TEST_START_TIME, toDate: TEST_END_TIME, - timeFieldName: '@timestamp', - field: { - name: 'bytes', - type: 'number', - }, + fieldName: 'bytes', }) .expect(200); diff --git a/x-pack/test/functional/es_archives/visualize/default/data.json b/x-pack/test/functional/es_archives/visualize/default/data.json index 26b033e28b4da..7a30079475aea 100644 --- a/x-pack/test/functional/es_archives/visualize/default/data.json +++ b/x-pack/test/functional/es_archives/visualize/default/data.json @@ -145,6 +145,30 @@ } } +{ + "type": "doc", + "value": { + "index": ".kibana", + "type": "doc", + "id": "index-pattern:logstash-2015.09.22", + "index": ".kibana_1", + "source": { + "index-pattern": { + "timeFieldName": "@timestamp", + "title": "logstash-2015.09.22", + "fields":"[{\"name\":\"scripted_date\",\"type\":\"date\",\"count\":0,\"scripted\":true,\"script\":\"1234\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"scripted_string\",\"type\":\"string\",\"count\":0,\"scripted\":true,\"script\":\"return 'hello'\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]" + }, + "migrationVersion": { + "index-pattern": "7.11.0" + }, + "references": [ + ], + "type": "index-pattern", + "updated_at": "2018-12-21T00:43:07.096Z" + } + } +} + { "type": "doc", "value": { From 70d62f7890d3dcbfef78c6e2a16d89d06e033b9a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Feb 2021 13:20:54 +0100 Subject: [PATCH 3/5] shadow ES level runtime fields --- x-pack/plugins/lens/server/routes/field_stats.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index 907a9c45b74f6..2c7896dd66103 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -87,8 +87,7 @@ export async function initFieldsRoute(setup: CoreSetup) { body: { query, aggs, - runtime_mappings: - !field.isMapped && field.runtimeField ? { [fieldName]: field.runtimeField } : {}, + runtime_mappings: field.runtimeField ? { [fieldName]: field.runtimeField } : {}, }, size: 0, }); From 4537e2ff0b6beec01e2c4d2f2131e40f86ab2e5c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 17 Feb 2021 09:33:06 +0100 Subject: [PATCH 4/5] add functional test --- .../api_integration/apis/lens/field_stats.ts | 27 +++++++++++++++++++ .../es_archives/visualize/default/data.json | 3 ++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts index 80cf5149f0496..af4a827193545 100644 --- a/x-pack/test/api_integration/apis/lens/field_stats.ts +++ b/x-pack/test/api_integration/apis/lens/field_stats.ts @@ -377,6 +377,33 @@ export default ({ getService }: FtrProviderContext) => { }); }); + it('should return top values for index pattern runtime string fields', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + fieldName: 'runtime_string_field', + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4634, + sampledDocuments: 4634, + sampledValues: 4634, + topValues: { + buckets: [ + { + count: 4634, + key: 'hello world!', + }, + ], + }, + }); + }); + it('should apply filters and queries', async () => { const { body } = await supertest .post('/api/lens/index_stats/logstash-2015.09.22/field') diff --git a/x-pack/test/functional/es_archives/visualize/default/data.json b/x-pack/test/functional/es_archives/visualize/default/data.json index 7a30079475aea..7d0ad0c25f96d 100644 --- a/x-pack/test/functional/es_archives/visualize/default/data.json +++ b/x-pack/test/functional/es_archives/visualize/default/data.json @@ -156,7 +156,8 @@ "index-pattern": { "timeFieldName": "@timestamp", "title": "logstash-2015.09.22", - "fields":"[{\"name\":\"scripted_date\",\"type\":\"date\",\"count\":0,\"scripted\":true,\"script\":\"1234\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"scripted_string\",\"type\":\"string\",\"count\":0,\"scripted\":true,\"script\":\"return 'hello'\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]" + "fields":"[{\"name\":\"scripted_date\",\"type\":\"date\",\"count\":0,\"scripted\":true,\"script\":\"1234\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"scripted_string\",\"type\":\"string\",\"count\":0,\"scripted\":true,\"script\":\"return 'hello'\",\"lang\":\"painless\",\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]", + "runtimeFieldMap":"{\"runtime_string_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit('hello world!')\"}}}" }, "migrationVersion": { "index-pattern": "7.11.0" From 2370e5e3f6d447326dee22d8e6328102bbf5bb00 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 17 Feb 2021 11:13:40 +0100 Subject: [PATCH 5/5] fix functional tests --- .../api_integration/apis/lens/field_stats.ts | 204 +++++++++--------- 1 file changed, 105 insertions(+), 99 deletions(-) diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts index b9cfcdac61b87..94960b9859121 100644 --- a/x-pack/test/api_integration/apis/lens/field_stats.ts +++ b/x-pack/test/api_integration/apis/lens/field_stats.ts @@ -22,16 +22,19 @@ export default ({ getService }: FtrProviderContext) => { describe('index stats apis', () => { before(async () => { await esArchiver.loadIfNeeded('logstash_functional'); - await esArchiver.loadIfNeeded('visualize/default'); - await esArchiver.loadIfNeeded('pre_calculated_histogram'); }); after(async () => { await esArchiver.unload('logstash_functional'); - await esArchiver.unload('visualize/default'); - await esArchiver.unload('pre_calculated_histogram'); }); describe('field distribution', () => { + before(async () => { + await esArchiver.loadIfNeeded('visualize/default'); + }); + after(async () => { + await esArchiver.unload('visualize/default'); + }); + it('should return a 404 for missing index patterns', async () => { await supertest .post('/api/lens/index_stats/123/field') @@ -327,101 +330,6 @@ export default ({ getService }: FtrProviderContext) => { }); }); - it('should return an auto histogram for precalculated histograms', async () => { - const { body } = await supertest - .post('/api/lens/index_stats/histogram-test/field') - .set(COMMON_HEADERS) - .send({ - dslQuery: { match_all: {} }, - fromDate: TEST_START_TIME, - toDate: TEST_END_TIME, - field: { - name: 'histogram-content', - type: 'histogram', - }, - }) - .expect(200); - - expect(body).to.eql({ - histogram: { - buckets: [ - { - count: 237, - key: 0, - }, - { - count: 323, - key: 0.47000000000000003, - }, - { - count: 454, - key: 0.9400000000000001, - }, - { - count: 166, - key: 1.4100000000000001, - }, - { - count: 168, - key: 1.8800000000000001, - }, - { - count: 425, - key: 2.35, - }, - { - count: 311, - key: 2.8200000000000003, - }, - { - count: 391, - key: 3.29, - }, - { - count: 406, - key: 3.7600000000000002, - }, - { - count: 324, - key: 4.23, - }, - { - count: 628, - key: 4.7, - }, - ], - }, - sampledDocuments: 7, - sampledValues: 3833, - totalDocuments: 7, - topValues: { buckets: [] }, - }); - }); - - it('should return a single-value histogram when filtering a precalculated histogram', async () => { - const { body } = await supertest - .post('/api/lens/index_stats/histogram-test/field') - .set(COMMON_HEADERS) - .send({ - dslQuery: { match: { 'histogram-title': 'single value' } }, - fromDate: TEST_START_TIME, - toDate: TEST_END_TIME, - field: { - name: 'histogram-content', - type: 'histogram', - }, - }) - .expect(200); - - expect(body).to.eql({ - histogram: { buckets: [{ count: 1, key: 1 }] }, - sampledDocuments: 1, - sampledValues: 1, - totalDocuments: 1, - topValues: { buckets: [] }, - }); - }); - it('should return histograms for scripted date fields', async () => { const { body } = await supertest .post('/api/lens/index_stats/logstash-2015.09.22/field') @@ -520,5 +428,103 @@ export default ({ getService }: FtrProviderContext) => { expect(body.totalDocuments).to.eql(425); }); }); + + describe('histogram', () => { + before(async () => { + await esArchiver.loadIfNeeded('pre_calculated_histogram'); + }); + after(async () => { + await esArchiver.unload('pre_calculated_histogram'); + }); + + it('should return an auto histogram for precalculated histograms', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/histogram-test/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + fieldName: 'histogram-content', + }) + .expect(200); + + expect(body).to.eql({ + histogram: { + buckets: [ + { + count: 237, + key: 0, + }, + { + count: 323, + key: 0.47000000000000003, + }, + { + count: 454, + key: 0.9400000000000001, + }, + { + count: 166, + key: 1.4100000000000001, + }, + { + count: 168, + key: 1.8800000000000001, + }, + { + count: 425, + key: 2.35, + }, + { + count: 311, + key: 2.8200000000000003, + }, + { + count: 391, + key: 3.29, + }, + { + count: 406, + key: 3.7600000000000002, + }, + { + count: 324, + key: 4.23, + }, + { + count: 628, + key: 4.7, + }, + ], + }, + sampledDocuments: 7, + sampledValues: 3833, + totalDocuments: 7, + topValues: { buckets: [] }, + }); + }); + + it('should return a single-value histogram when filtering a precalculated histogram', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/histogram-test/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match: { 'histogram-title': 'single value' } }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + fieldName: 'histogram-content', + }) + .expect(200); + + expect(body).to.eql({ + histogram: { buckets: [{ count: 1, key: 1 }] }, + sampledDocuments: 1, + sampledValues: 1, + totalDocuments: 1, + topValues: { buckets: [] }, + }); + }); + }); }); };