[Lens] Adds using queries/filters for field existence endpoint

x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx

@@ -206,6 +206,9 @@ const initialState: IndexPatternPrivateState = {
     },
   },
 };
+
+const dslQuery = { bool: { must: [{ match_all: {} }], filter: [], should: [], must_not: [] } };
+
 describe('IndexPattern Data Panel', () => {
   let defaultProps: Parameters<typeof InnerIndexPatternDataPanel>[0];
   let core: ReturnType<typeof coreMock['createSetup']>;
@@ -271,8 +274,8 @@ describe('IndexPattern Data Panel', () => {
   describe('loading existence data', () => {
     function testProps() {
       const setState = jest.fn();
-      core.http.get.mockImplementation(async ({ path }) => {
-        const parts = path.split('/');
+      core.http.post.mockImplementation(async path => {
+        const parts = ((path as unknown) as string).split('/');
         const indexPatternTitle = parts[parts.length - 1];
         return {
           indexPatternTitle: `${indexPatternTitle}_testtitle`,
@@ -385,24 +388,24 @@ describe('IndexPattern Data Panel', () => {
       });
 
       expect(setState).toHaveBeenCalledTimes(2);
-      expect(core.http.get).toHaveBeenCalledTimes(2);
+      expect(core.http.post).toHaveBeenCalledTimes(2);
 
-      expect(core.http.get).toHaveBeenCalledWith({
-        path: '/api/lens/existing_fields/a',
-        query: {
+      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
+        body: JSON.stringify({
+          dslQuery,
           fromDate: '2019-01-01',
           toDate: '2020-01-01',
           timeFieldName: 'atime',
-        },
+        }),
       });
 
-      expect(core.http.get).toHaveBeenCalledWith({
-        path: '/api/lens/existing_fields/a',
-        query: {
+      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
+        body: JSON.stringify({
+          dslQuery,
           fromDate: '2019-01-01',
           toDate: '2020-01-02',
           timeFieldName: 'atime',
-        },
+        }),
       });
 
       const nextState = setState.mock.calls[1][0]({
@@ -428,22 +431,22 @@ describe('IndexPattern Data Panel', () => {
 
       expect(setState).toHaveBeenCalledTimes(2);
 
-      expect(core.http.get).toHaveBeenCalledWith({
-        path: '/api/lens/existing_fields/a',
-        query: {
+      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
+        body: JSON.stringify({
+          dslQuery,
           fromDate: '2019-01-01',
           toDate: '2020-01-01',
           timeFieldName: 'atime',
-        },
+        }),
       });
 
-      expect(core.http.get).toHaveBeenCalledWith({
-        path: '/api/lens/existing_fields/b',
-        query: {
+      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/b', {
+        body: JSON.stringify({
+          dslQuery,
           fromDate: '2019-01-01',
           toDate: '2020-01-01',
           timeFieldName: 'btime',
-        },
+        }),
       });
 
       const nextState = setState.mock.calls[1][0]({
@@ -476,13 +479,13 @@ describe('IndexPattern Data Panel', () => {
       let overlapCount = 0;
       const props = testProps();
 
-      core.http.get.mockImplementation(({ path }) => {
+      core.http.post.mockImplementation(path => {
         if (queryCount) {
           ++overlapCount;
         }
         ++queryCount;
 
-        const parts = path.split('/');
+        const parts = ((path as unknown) as string).split('/');
         const indexPatternTitle = parts[parts.length - 1];
         const result = Promise.resolve({
           indexPatternTitle,
@@ -516,7 +519,7 @@ describe('IndexPattern Data Panel', () => {
         inst.update();
       });
 
-      expect(core.http.get).toHaveBeenCalledTimes(2);
+      expect(core.http.post).toHaveBeenCalledTimes(2);
       expect(overlapCount).toEqual(0);
     });
   });

x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.tsx

@@ -40,6 +40,7 @@ import { trackUiEvent } from '../lens_ui_telemetry';
 import { syncExistingFields } from './loader';
 import { fieldExists } from './pure_helpers';
 import { Loader } from '../loader';
+import { esQuery, IIndexPattern } from '../../../../../../src/plugins/data/public';
 
 export type Props = DatasourceDataPanelProps<IndexPatternPrivateState> & {
   changeIndexPattern: (
@@ -113,6 +114,13 @@ export function IndexPatternDataPanel({
       timeFieldName: indexPatterns[id].timeFieldName,
     }));
 
+  const dslQuery = esQuery.buildEsQuery(
+    indexPatterns[currentIndexPatternId] as IIndexPattern,
+    query,
+    filters,
+    esQuery.getEsQueryConfig(core.uiSettings)
+  );
+
   return (
     <>
       <Loader
@@ -121,10 +129,13 @@ export function IndexPatternDataPanel({
             dateRange,
             setState,
             indexPatterns: indexPatternList,
-            fetchJson: core.http.get,
+            fetchJson: core.http.post,
+            dslQuery,
           })
         }
         loadDeps={[
+          query,
+          filters,
           dateRange.fromDate,
           dateRange.toDate,
           indexPatternList.map(x => `${x.title}:${x.timeFieldName}`).join(','),

x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.test.ts

@@ -535,9 +535,18 @@ describe('loader', () => {
   });
 
   describe('syncExistingFields', () => {
+    const dslQuery = {
+      bool: {
+        must: [],
+        filter: [{ match_all: {} }],
+        should: [],
+        must_not: [],
+      },
+    };
+
     it('should call once for each index pattern', async () => {
       const setState = jest.fn();
-      const fetchJson = jest.fn(({ path }: { path: string }) => {
+      const fetchJson = jest.fn((path: string) => {
         const indexPatternTitle = _.last(path.split('/'));
         return {
           indexPatternTitle,
@@ -553,6 +562,7 @@ describe('loader', () => {
         fetchJson: fetchJson as any,
         indexPatterns: [{ id: 'a' }, { id: 'b' }, { id: 'c' }],
         setState,
+        dslQuery,
       });
 
       expect(fetchJson).toHaveBeenCalledTimes(3);

x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.ts

@@ -215,26 +215,28 @@ export async function syncExistingFields({
   dateRange,
   fetchJson,
   setState,
+  dslQuery,
 }: {
   dateRange: DateRange;
   indexPatterns: Array<{ id: string; timeFieldName?: string | null }>;
-  fetchJson: HttpSetup['get'];
+  fetchJson: HttpSetup['post'];
   setState: SetState;
+  dslQuery: object;
 }) {
   const emptinessInfo = await Promise.all(
     indexPatterns.map(pattern => {
-      const query: Record<string, string> = {
+      const body: Record<string, string | object> = {
+        dslQuery,
         fromDate: dateRange.fromDate,
         toDate: dateRange.toDate,
       };
 
       if (pattern.timeFieldName) {
-        query.timeFieldName = pattern.timeFieldName;
+        body.timeFieldName = pattern.timeFieldName;
       }
 
-      return fetchJson({
-        path: `${BASE_API_URL}/existing_fields/${pattern.id}`,
-        query,
+      return fetchJson(`${BASE_API_URL}/existing_fields/${pattern.id}`, {
+        body: JSON.stringify(body),
       }) as Promise<ExistingFields>;
     })
   );

x-pack/plugins/lens/server/routes/existing_fields.ts

@@ -46,14 +46,16 @@ const metaFields = ['_source', '_id', '_type', '_index', '_score'];
 
 export async function existingFieldsRoute(setup: CoreSetup) {
   const router = setup.http.createRouter();
-  router.get(
+
+  router.post(
     {
       path: `${BASE_API_URL}/existing_fields/{indexPatternId}`,
       validate: {
         params: schema.object({
           indexPatternId: schema.string(),
         }),
-        query: schema.object({
+        body: schema.object({
+          dslQuery: schema.object({}, { allowUnknowns: true }),
           fromDate: schema.maybe(schema.string()),
           toDate: schema.maybe(schema.string()),
           timeFieldName: schema.maybe(schema.string()),
@@ -64,8 +66,8 @@ export async function existingFieldsRoute(setup: CoreSetup) {
       try {
         return res.ok({
           body: await fetchFieldExistence({
-            ...req.query,
             ...req.params,
+            ...req.body,
             context,
           }),
         });
@@ -91,12 +93,14 @@ export async function existingFieldsRoute(setup: CoreSetup) {
 async function fetchFieldExistence({
   context,
   indexPatternId,
+  dslQuery = { match_all: {} },
   fromDate,
   toDate,
   timeFieldName,
 }: {
   indexPatternId: string;
   context: RequestHandlerContext;
+  dslQuery: object;
   fromDate?: string;
   toDate?: string;
   timeFieldName?: string;
@@ -109,10 +113,10 @@ async function fetchFieldExistence({
   } = await fetchIndexPatternDefinition(indexPatternId, context);
 
   const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
-
   const docs = await fetchIndexPatternStats({
     fromDate,
     toDate,
+    dslQuery,
     client: context.core.elasticsearch.dataClient,
     index: indexPatternTitle,
     timeFieldName: timeFieldName || indexPattern.attributes.timeFieldName,
@@ -197,24 +201,23 @@ export function buildFieldList(
 async function fetchIndexPatternStats({
   client,
   index,
+  dslQuery,
   timeFieldName,
   fromDate,
   toDate,
   fields,
 }: {
   client: IScopedClusterClient;
   index: string;
+  dslQuery: object;
   timeFieldName?: string;
   fromDate?: string;
   toDate?: string;
   fields: Field[];
 }) {
-  let query;
-
-  if (timeFieldName && fromDate && toDate) {
-    query = {
-      bool: {
-        filter: [
+  const filter =
+    timeFieldName && fromDate && toDate
+      ? [
           {
             range: {
               [timeFieldName]: {
@@ -223,16 +226,17 @@ async function fetchIndexPatternStats({
               },
             },
           },
-        ],
-      },
-    };
-  } else {
-    query = {
-      match_all: {},
-    };
-  }
-  const scriptedFields = fields.filter(f => f.isScript);
+          dslQuery,
+        ]
+      : [dslQuery];
+
+  const query = {
+    bool: {
+      filter,
+    },
+  };
 
+  const scriptedFields = fields.filter(f => f.isScript);
   const result = await client.callAsCurrentUser('search', {
     index,
     body: {
@@ -251,7 +255,6 @@ async function fetchIndexPatternStats({
       }, {} as Record<string, unknown>),
     },
   });
-
   return result.hits.hits;
 }