diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx index c669369e6e1d0..3066ac0e11325 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx @@ -206,6 +206,9 @@ const initialState: IndexPatternPrivateState = { }, }, }; + +const dslQuery = { bool: { must: [{ match_all: {} }], filter: [], should: [], must_not: [] } }; + describe('IndexPattern Data Panel', () => { let defaultProps: Parameters[0]; let core: ReturnType; @@ -271,8 +274,8 @@ describe('IndexPattern Data Panel', () => { describe('loading existence data', () => { function testProps() { const setState = jest.fn(); - core.http.get.mockImplementation(async ({ path }) => { - const parts = path.split('/'); + core.http.post.mockImplementation(async path => { + const parts = ((path as unknown) as string).split('/'); const indexPatternTitle = parts[parts.length - 1]; return { indexPatternTitle: `${indexPatternTitle}_testtitle`, @@ -385,24 +388,24 @@ describe('IndexPattern Data Panel', () => { }); expect(setState).toHaveBeenCalledTimes(2); - expect(core.http.get).toHaveBeenCalledTimes(2); + expect(core.http.post).toHaveBeenCalledTimes(2); - expect(core.http.get).toHaveBeenCalledWith({ - path: '/api/lens/existing_fields/a', - query: { + expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', { + body: JSON.stringify({ + dslQuery, fromDate: '2019-01-01', toDate: '2020-01-01', timeFieldName: 'atime', - }, + }), }); - expect(core.http.get).toHaveBeenCalledWith({ - path: '/api/lens/existing_fields/a', - query: { + expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', { + body: JSON.stringify({ + dslQuery, fromDate: '2019-01-01', toDate: '2020-01-02', timeFieldName: 'atime', - }, + }), }); const nextState = setState.mock.calls[1][0]({ @@ -428,22 +431,22 @@ describe('IndexPattern Data Panel', () => { expect(setState).toHaveBeenCalledTimes(2); - expect(core.http.get).toHaveBeenCalledWith({ - path: '/api/lens/existing_fields/a', - query: { + expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', { + body: JSON.stringify({ + dslQuery, fromDate: '2019-01-01', toDate: '2020-01-01', timeFieldName: 'atime', - }, + }), }); - expect(core.http.get).toHaveBeenCalledWith({ - path: '/api/lens/existing_fields/b', - query: { + expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/b', { + body: JSON.stringify({ + dslQuery, fromDate: '2019-01-01', toDate: '2020-01-01', timeFieldName: 'btime', - }, + }), }); const nextState = setState.mock.calls[1][0]({ @@ -476,13 +479,13 @@ describe('IndexPattern Data Panel', () => { let overlapCount = 0; const props = testProps(); - core.http.get.mockImplementation(({ path }) => { + core.http.post.mockImplementation(path => { if (queryCount) { ++overlapCount; } ++queryCount; - const parts = path.split('/'); + const parts = ((path as unknown) as string).split('/'); const indexPatternTitle = parts[parts.length - 1]; const result = Promise.resolve({ indexPatternTitle, @@ -516,7 +519,7 @@ describe('IndexPattern Data Panel', () => { inst.update(); }); - expect(core.http.get).toHaveBeenCalledTimes(2); + expect(core.http.post).toHaveBeenCalledTimes(2); expect(overlapCount).toEqual(0); }); }); diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.tsx index 1b86c07a31c11..7a3c04b67fbc4 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.tsx @@ -40,6 +40,7 @@ import { trackUiEvent } from '../lens_ui_telemetry'; import { syncExistingFields } from './loader'; import { fieldExists } from './pure_helpers'; import { Loader } from '../loader'; +import { esQuery, IIndexPattern } from '../../../../../../src/plugins/data/public'; export type Props = DatasourceDataPanelProps & { changeIndexPattern: ( @@ -113,6 +114,13 @@ export function IndexPatternDataPanel({ timeFieldName: indexPatterns[id].timeFieldName, })); + const dslQuery = esQuery.buildEsQuery( + indexPatterns[currentIndexPatternId] as IIndexPattern, + query, + filters, + esQuery.getEsQueryConfig(core.uiSettings) + ); + return ( <> `${x.title}:${x.timeFieldName}`).join(','), diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.test.ts b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.test.ts index f8961c30d14ee..ea9c8213ba909 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.test.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.test.ts @@ -535,9 +535,18 @@ describe('loader', () => { }); describe('syncExistingFields', () => { + const dslQuery = { + bool: { + must: [], + filter: [{ match_all: {} }], + should: [], + must_not: [], + }, + }; + it('should call once for each index pattern', async () => { const setState = jest.fn(); - const fetchJson = jest.fn(({ path }: { path: string }) => { + const fetchJson = jest.fn((path: string) => { const indexPatternTitle = _.last(path.split('/')); return { indexPatternTitle, @@ -553,6 +562,7 @@ describe('loader', () => { fetchJson: fetchJson as any, indexPatterns: [{ id: 'a' }, { id: 'b' }, { id: 'c' }], setState, + dslQuery, }); expect(fetchJson).toHaveBeenCalledTimes(3); diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.ts b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.ts index ed3d8a91b366d..f4d5857f4826d 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.ts +++ b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.ts @@ -215,26 +215,28 @@ export async function syncExistingFields({ dateRange, fetchJson, setState, + dslQuery, }: { dateRange: DateRange; indexPatterns: Array<{ id: string; timeFieldName?: string | null }>; - fetchJson: HttpSetup['get']; + fetchJson: HttpSetup['post']; setState: SetState; + dslQuery: object; }) { const emptinessInfo = await Promise.all( indexPatterns.map(pattern => { - const query: Record = { + const body: Record = { + dslQuery, fromDate: dateRange.fromDate, toDate: dateRange.toDate, }; if (pattern.timeFieldName) { - query.timeFieldName = pattern.timeFieldName; + body.timeFieldName = pattern.timeFieldName; } - return fetchJson({ - path: `${BASE_API_URL}/existing_fields/${pattern.id}`, - query, + return fetchJson(`${BASE_API_URL}/existing_fields/${pattern.id}`, { + body: JSON.stringify(body), }) as Promise; }) ); diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index 40b2766a647cf..57c1680413537 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -46,14 +46,16 @@ const metaFields = ['_source', '_id', '_type', '_index', '_score']; export async function existingFieldsRoute(setup: CoreSetup) { const router = setup.http.createRouter(); - router.get( + + router.post( { path: `${BASE_API_URL}/existing_fields/{indexPatternId}`, validate: { params: schema.object({ indexPatternId: schema.string(), }), - query: schema.object({ + body: schema.object({ + dslQuery: schema.object({}, { allowUnknowns: true }), fromDate: schema.maybe(schema.string()), toDate: schema.maybe(schema.string()), timeFieldName: schema.maybe(schema.string()), @@ -64,8 +66,8 @@ export async function existingFieldsRoute(setup: CoreSetup) { try { return res.ok({ body: await fetchFieldExistence({ - ...req.query, ...req.params, + ...req.body, context, }), }); @@ -91,12 +93,14 @@ export async function existingFieldsRoute(setup: CoreSetup) { async function fetchFieldExistence({ context, indexPatternId, + dslQuery = { match_all: {} }, fromDate, toDate, timeFieldName, }: { indexPatternId: string; context: RequestHandlerContext; + dslQuery: object; fromDate?: string; toDate?: string; timeFieldName?: string; @@ -109,10 +113,10 @@ async function fetchFieldExistence({ } = await fetchIndexPatternDefinition(indexPatternId, context); const fields = buildFieldList(indexPattern, mappings, fieldDescriptors); - const docs = await fetchIndexPatternStats({ fromDate, toDate, + dslQuery, client: context.core.elasticsearch.dataClient, index: indexPatternTitle, timeFieldName: timeFieldName || indexPattern.attributes.timeFieldName, @@ -197,6 +201,7 @@ export function buildFieldList( async function fetchIndexPatternStats({ client, index, + dslQuery, timeFieldName, fromDate, toDate, @@ -204,17 +209,15 @@ async function fetchIndexPatternStats({ }: { client: IScopedClusterClient; index: string; + dslQuery: object; timeFieldName?: string; fromDate?: string; toDate?: string; fields: Field[]; }) { - let query; - - if (timeFieldName && fromDate && toDate) { - query = { - bool: { - filter: [ + const filter = + timeFieldName && fromDate && toDate + ? [ { range: { [timeFieldName]: { @@ -223,16 +226,17 @@ async function fetchIndexPatternStats({ }, }, }, - ], - }, - }; - } else { - query = { - match_all: {}, - }; - } - const scriptedFields = fields.filter(f => f.isScript); + dslQuery, + ] + : [dslQuery]; + + const query = { + bool: { + filter, + }, + }; + const scriptedFields = fields.filter(f => f.isScript); const result = await client.callAsCurrentUser('search', { index, body: { @@ -251,7 +255,6 @@ async function fetchIndexPatternStats({ }, {} as Record), }, }); - return result.hits.hits; } diff --git a/x-pack/test/api_integration/apis/lens/existing_fields.ts b/x-pack/test/api_integration/apis/lens/existing_fields.ts index cf65e114d3896..b3810cf468b55 100644 --- a/x-pack/test/api_integration/apis/lens/existing_fields.ts +++ b/x-pack/test/api_integration/apis/lens/existing_fields.ts @@ -8,8 +8,8 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; -const TEST_START_TIME = encodeURIComponent('2015-09-19T06:31:44.000'); -const TEST_END_TIME = encodeURIComponent('2015-09-23T18:31:44.000'); +const TEST_START_TIME = '2015-09-19T06:31:44.000'; +const TEST_END_TIME = '2015-09-23T18:31:44.000'; const COMMON_HEADERS = { 'kbn-xsrf': 'some-xsrf-token', }; @@ -147,12 +147,17 @@ export default ({ getService }: FtrProviderContext) => { describe('existence', () => { it('should find which fields exist in the sample documents', async () => { const { body } = await supertest - .get( - `/api/lens/existing_fields/${encodeURIComponent( - 'logstash-*' - )}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}` - ) + .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`) .set(COMMON_HEADERS) + .send({ + dslQuery: { + bool: { + filter: [{ match_all: {} }], + }, + }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + }) .expect(200); expect(body.indexPatternTitle).to.eql('logstash-*'); @@ -161,25 +166,67 @@ export default ({ getService }: FtrProviderContext) => { it('should succeed for thousands of fields', async () => { const { body } = await supertest - .get( - `/api/lens/existing_fields/${encodeURIComponent( - 'metricbeat-*' - )}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}` - ) + .post(`/api/lens/existing_fields/${encodeURIComponent('metricbeat-*')}`) .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + }) .expect(200); expect(body.indexPatternTitle).to.eql('metricbeat-*'); expect(body.existingFieldNames.sort()).to.eql(metricBeatData.sort()); }); - it('should throw a 404 for a non-existent index', async () => { - await supertest - .get( - `/api/lens/existing_fields/nadachance?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}` - ) + it('should return fields filtered by query and filters', async () => { + const expectedFieldNames = [ + '@message', + '@message.raw', + '@tags', + '@tags.raw', + '@timestamp', + 'agent', + 'agent.raw', + 'bytes', + 'clientip', + 'extension', + 'extension.raw', + 'headings', + 'headings.raw', + 'host', + 'host.raw', + 'index', + 'index.raw', + 'referer', + 'request', + 'request.raw', + 'response', + 'response.raw', + 'spaces', + 'spaces.raw', + 'type', + 'url', + 'url.raw', + 'utc_time', + 'xss', + 'xss.raw', + ]; + + const { body } = await supertest + .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`) .set(COMMON_HEADERS) - .expect(404); + .send({ + dslQuery: { + bool: { + filter: [{ match: { referer: 'https://www.taylorswift.com/' } }], + }, + }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + }) + .expect(200); + expect(body.existingFieldNames.sort()).to.eql(expectedFieldNames.sort()); }); }); }); diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts index 51d81668d275d..2d394e35725c2 100644 --- a/x-pack/test/api_integration/apis/lens/field_stats.ts +++ b/x-pack/test/api_integration/apis/lens/field_stats.ts @@ -62,7 +62,7 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - expect(body).to.have.property('totalDocuments', 4633); + expect(body).to.have.property('totalDocuments', 4634); }); it('should return an auto histogram for numbers and top values', async () => { @@ -82,9 +82,9 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); expect(body).to.eql({ - totalDocuments: 4633, - sampledDocuments: 4633, - sampledValues: 4633, + totalDocuments: 4634, + sampledDocuments: 4634, + sampledValues: 4634, histogram: { buckets: [ { @@ -96,7 +96,7 @@ export default ({ getService }: FtrProviderContext) => { key: 1999, }, { - count: 885, + count: 886, key: 3998, }, { @@ -139,6 +139,10 @@ export default ({ getService }: FtrProviderContext) => { count: 5, key: 3954, }, + { + count: 5, + key: 5846, + }, { count: 5, key: 6497, @@ -159,10 +163,6 @@ export default ({ getService }: FtrProviderContext) => { count: 4, key: 4669, }, - { - count: 4, - key: 5846, - }, { count: 4, key: 5863, @@ -193,11 +193,11 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); expect(body).to.eql({ - totalDocuments: 4633, + totalDocuments: 4634, histogram: { buckets: [ { - count: 1161, + count: 1162, key: 1442875680000, }, { @@ -230,8 +230,8 @@ export default ({ getService }: FtrProviderContext) => { .expect(200); expect(body).to.eql({ - totalDocuments: 4633, - sampledDocuments: 4633, + totalDocuments: 4634, + sampledDocuments: 4634, sampledValues: 4633, topValues: { buckets: [ diff --git a/x-pack/test/functional/es_archives/logstash_functional/data.json.gz b/x-pack/test/functional/es_archives/logstash_functional/data.json.gz index 2cef071738526..02195f888b93c 100644 Binary files a/x-pack/test/functional/es_archives/logstash_functional/data.json.gz and b/x-pack/test/functional/es_archives/logstash_functional/data.json.gz differ