[Partial Results] Move other bucket into Search Source

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md → docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.inspector.md

@@ -1,11 +1,13 @@
 <!-- Do not edit this file. It is automatically generated by API Documenter. -->
 
-[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) &gt; [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md)
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) &gt; [inspector](./kibana-plugin-plugins-data-public.isearchoptions.inspector.md)
 
-## ISearchOptions.requestResponder property
+## ISearchOptions.inspector property
+
+Inspector integration options
 
 <b>Signature:</b>
 
 ```typescript
-requestResponder?: RequestResponder;
+inspector?: IInspectorInfo;
 ```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md

@@ -16,10 +16,10 @@ export interface ISearchOptions
 |  --- | --- | --- |
 |  [abortSignal](./kibana-plugin-plugins-data-public.isearchoptions.abortsignal.md) | <code>AbortSignal</code> | An <code>AbortSignal</code> that allows the caller of <code>search</code> to abort a search request. |
 |  [indexPattern](./kibana-plugin-plugins-data-public.isearchoptions.indexpattern.md) | <code>IndexPattern</code> | Index pattern reference is used for better error messages |
+|  [inspector](./kibana-plugin-plugins-data-public.isearchoptions.inspector.md) | <code>IInspectorInfo</code> | Inspector integration options |
 |  [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
 |  [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
-|  [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) | <code>RequestResponder</code> |  |
 |  [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch.md

@@ -14,7 +14,7 @@ Fetch this source and reject the returned Promise on error
 <b>Signature:</b>
 
 ```typescript
-fetch(options?: ISearchOptions): Promise<import("@elastic/elasticsearch/api/types").SearchResponse<any>>;
+fetch(options?: ISearchOptions): Promise<estypes.SearchResponse<any>>;
 ```
 
 ## Parameters
@@ -25,5 +25,5 @@ fetch(options?: ISearchOptions): Promise<import("@elastic/elasticsearch/api/type
 
 <b>Returns:</b>
 
-`Promise<import("@elastic/elasticsearch/api/types").SearchResponse<any>>`
+`Promise<estypes.SearchResponse<any>>`
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch_.md

@@ -9,7 +9,7 @@ Fetch this source from Elasticsearch, returning an observable over the response(
 <b>Signature:</b>
 
 ```typescript
-fetch$(options?: ISearchOptions): import("rxjs").Observable<import("@elastic/elasticsearch/api/types").SearchResponse<any>>;
+fetch$(options?: ISearchOptions): Observable<estypes.SearchResponse<any>>;
 ```
 
 ## Parameters
@@ -20,5 +20,5 @@ fetch$(options?: ISearchOptions): import("rxjs").Observable<import("@elastic/ela
 
 <b>Returns:</b>
 
-`import("rxjs").Observable<import("@elastic/elasticsearch/api/types").SearchResponse<any>>`
+`Observable<estypes.SearchResponse<any>>`
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.aggs.md

@@ -9,5 +9,5 @@
 <b>Signature:</b>
 
 ```typescript
-aggs?: any;
+aggs?: object | IAggConfigs | (() => object);
 ```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md

@@ -16,7 +16,7 @@ export interface SearchSourceFields
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [aggs](./kibana-plugin-plugins-data-public.searchsourcefields.aggs.md) | <code>any</code> | [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) |
+|  [aggs](./kibana-plugin-plugins-data-public.searchsourcefields.aggs.md) | <code>object &#124; IAggConfigs &#124; (() =&gt; object)</code> | [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) |
 |  [fields](./kibana-plugin-plugins-data-public.searchsourcefields.fields.md) | <code>SearchFieldValue[]</code> | Retrieve fields via the search Fields API |
 |  [fieldsFromSource](./kibana-plugin-plugins-data-public.searchsourcefields.fieldsfromsource.md) | <code>NameList</code> | Retreive fields directly from \_source (legacy behavior) |
 |  [filter](./kibana-plugin-plugins-data-public.searchsourcefields.filter.md) | <code>Filter[] &#124; Filter &#124; (() =&gt; Filter[] &#124; Filter &#124; undefined)</code> | [Filter](./kibana-plugin-plugins-data-public.filter.md) |

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsserviceprovider.start.md

@@ -8,7 +8,7 @@
 
 ```typescript
 start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps): {
-        indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "update" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
+        indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "update" | "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
     };
 ```
 
@@ -22,6 +22,6 @@ start(core: CoreStart, { fieldFormats, logger }: IndexPatternsServiceStartDeps):
 <b>Returns:</b>
 
 `{
-        indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "update" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
+        indexPatternsServiceFactory: (savedObjectsClient: Pick<import("../../../../core/server").SavedObjectsClient, "update" | "get" | "delete" | "create" | "bulkCreate" | "checkConflicts" | "find" | "bulkGet" | "resolve" | "addToNamespaces" | "deleteFromNamespaces" | "bulkUpdate" | "removeReferencesTo" | "openPointInTimeForType" | "closePointInTime" | "createPointInTimeFinder" | "errors">, elasticsearchClient: ElasticsearchClient) => Promise<IndexPatternsCommonService>;
     }`
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md → docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.inspector.md

@@ -1,11 +1,13 @@
 <!-- Do not edit this file. It is automatically generated by API Documenter. -->
 
-[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) &gt; [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md)
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) &gt; [inspector](./kibana-plugin-plugins-data-server.isearchoptions.inspector.md)
 
-## ISearchOptions.requestResponder property
+## ISearchOptions.inspector property
+
+Inspector integration options
 
 <b>Signature:</b>
 
 ```typescript
-requestResponder?: RequestResponder;
+inspector?: IInspectorInfo;
 ```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md

@@ -16,10 +16,10 @@ export interface ISearchOptions
 |  --- | --- | --- |
 |  [abortSignal](./kibana-plugin-plugins-data-server.isearchoptions.abortsignal.md) | <code>AbortSignal</code> | An <code>AbortSignal</code> that allows the caller of <code>search</code> to abort a search request. |
 |  [indexPattern](./kibana-plugin-plugins-data-server.isearchoptions.indexpattern.md) | <code>IndexPattern</code> | Index pattern reference is used for better error messages |
+|  [inspector](./kibana-plugin-plugins-data-server.isearchoptions.inspector.md) | <code>IInspectorInfo</code> | Inspector integration options |
 |  [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
 |  [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
-|  [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) | <code>RequestResponder</code> |  |
 |  [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

examples/search_examples/public/search/app.tsx

@@ -195,13 +195,10 @@ export const SearchExamplesApp = ({
         .setField('trackTotalHits', 100);
 
       if (selectedNumericField) {
-        searchSource.setField('aggs', () => {
-          return data.search.aggs
-            .createAggConfigs(indexPattern, [
-              { type: 'avg', params: { field: selectedNumericField.name } },
-            ])
-            .toDsl();
-        });
+        const ac = data.search.aggs.createAggConfigs(indexPattern, [
+          { type: 'avg', params: { field: selectedNumericField.name } },
+        ]);
+        searchSource.setField('aggs', ac);
       }
 
       setRequest(searchSource.getSearchRequestBody());

src/plugins/data/common/search/aggs/agg_type.ts

@@ -13,12 +13,23 @@ import { ISearchSource } from 'src/plugins/data/public';
 import { DatatableColumnType, SerializedFieldFormat } from 'src/plugins/expressions/common';
 import type { RequestAdapter } from 'src/plugins/inspector/common';
 
+import { estypes } from '@elastic/elasticsearch';
 import { initParams } from './agg_params';
 import { AggConfig } from './agg_config';
 import { IAggConfigs } from './agg_configs';
 import { BaseParamType } from './param_types/base';
 import { AggParamType } from './param_types/agg';
 
+type PostFlightRequestFn<TAggConfig> = (
+  resp: estypes.SearchResponse<any>,
+  aggConfigs: IAggConfigs,
+  aggConfig: TAggConfig,
+  searchSource: ISearchSource,
+  inspectorRequestAdapter?: RequestAdapter,
+  abortSignal?: AbortSignal,
+  searchSessionId?: string
+) => Promise<estypes.SearchResponse<any>>;
+
 export interface AggTypeConfig<
   TAggConfig extends AggConfig = AggConfig,
   TParam extends AggParamType<TAggConfig> = AggParamType<TAggConfig>
@@ -40,15 +51,7 @@ export interface AggTypeConfig<
   customLabels?: boolean;
   json?: boolean;
   decorateAggConfig?: () => any;
-  postFlightRequest?: (
-    resp: any,
-    aggConfigs: IAggConfigs,
-    aggConfig: TAggConfig,
-    searchSource: ISearchSource,
-    inspectorRequestAdapter?: RequestAdapter,
-    abortSignal?: AbortSignal,
-    searchSessionId?: string
-  ) => Promise<any>;
+  postFlightRequest?: PostFlightRequestFn<TAggConfig>;
   getSerializedFormat?: (agg: TAggConfig) => SerializedFieldFormat;
   getValue?: (agg: TAggConfig, bucket: any) => any;
   getKey?: (bucket: any, key: any, agg: TAggConfig) => any;
@@ -188,15 +191,7 @@ export class AggType<
    * @param searchSessionId - searchSessionId to be used for grouping requests into a single search session
    * @return {Promise}
    */
-  postFlightRequest: (
-    resp: any,
-    aggConfigs: IAggConfigs,
-    aggConfig: TAggConfig,
-    searchSource: ISearchSource,
-    inspectorRequestAdapter?: RequestAdapter,
-    abortSignal?: AbortSignal,
-    searchSessionId?: string
-  ) => Promise<any>;
+  postFlightRequest: PostFlightRequestFn<TAggConfig>;
   /**
    * Get the serialized format for the values produced by this agg type,
    * overridden by several metrics that always output a simple number.

src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts

@@ -433,7 +433,7 @@ describe('Terms Agg Other bucket helper', () => {
           aggConfigs.aggs[0] as IBucketAggConfig,
           otherAggConfig()
         );
-        expect(mergedResponse.aggregations['1'].buckets[3].key).toEqual('__other__');
+        expect((mergedResponse!.aggregations!['1'] as any).buckets[3].key).toEqual('__other__');
       }
     });
 
@@ -455,7 +455,7 @@ describe('Terms Agg Other bucket helper', () => {
           otherAggConfig()
         );
 
-        expect(mergedResponse.aggregations['1'].buckets[1]['2'].buckets[3].key).toEqual(
+        expect((mergedResponse!.aggregations!['1'] as any).buckets[1]['2'].buckets[3].key).toEqual(
           '__other__'
         );
       }
@@ -471,7 +471,7 @@ describe('Terms Agg Other bucket helper', () => {
         aggConfigs.aggs[0] as IBucketAggConfig
       );
       expect(
-        updatedResponse.aggregations['1'].buckets.find(
+        (updatedResponse!.aggregations!['1'] as any).buckets.find(
           (bucket: Record<string, any>) => bucket.key === '__missing__'
         )
       ).toBeDefined();

src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts

@@ -7,6 +7,7 @@
  */
 
 import { isNumber, keys, values, find, each, cloneDeep, flatten } from 'lodash';
+import { estypes } from '@elastic/elasticsearch';
 import { buildExistsFilter, buildPhrasesFilter, buildQueryFromFilters } from '../../../../common';
 import { AggGroupNames } from '../agg_groups';
 import { IAggConfigs } from '../agg_configs';
@@ -42,7 +43,7 @@ const getNestedAggDSL = (aggNestedDsl: Record<string, any>, startFromAggId: stri
  */
 const getAggResultBuckets = (
   aggConfigs: IAggConfigs,
-  response: any,
+  response: estypes.SearchResponse<any>['aggregations'],
   aggWithOtherBucket: IBucketAggConfig,
   key: string
 ) => {
@@ -72,8 +73,8 @@ const getAggResultBuckets = (
       }
     }
   }
-  if (responseAgg[aggWithOtherBucket.id]) {
-    return responseAgg[aggWithOtherBucket.id].buckets;
+  if (responseAgg?.[aggWithOtherBucket.id]) {
+    return (responseAgg[aggWithOtherBucket.id] as any).buckets;
   }
   return [];
 };
@@ -235,11 +236,11 @@ export const buildOtherBucketAgg = (
 
 export const mergeOtherBucketAggResponse = (
   aggsConfig: IAggConfigs,
-  response: any,
+  response: estypes.SearchResponse<any>,
   otherResponse: any,
   otherAgg: IBucketAggConfig,
   requestAgg: Record<string, any>
-) => {
+): estypes.SearchResponse<any> => {
   const updatedResponse = cloneDeep(response);
   each(otherResponse.aggregations['other-filter'].buckets, (bucket, key) => {
     if (!bucket.doc_count || key === undefined) return;
@@ -276,7 +277,7 @@ export const mergeOtherBucketAggResponse = (
 };
 
 export const updateMissingBucket = (
-  response: any,
+  response: estypes.SearchResponse<any>,
   aggConfigs: IAggConfigs,
   agg: IBucketAggConfig
 ) => {

src/plugins/data/common/search/aggs/buckets/terms.ts

@@ -101,25 +101,21 @@ export const getTermsBucketAgg = () =>
 
         nestedSearchSource.setField('aggs', filterAgg);
 
-        const requestResponder = inspectorRequestAdapter?.start(
-          i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', {
-            defaultMessage: 'Other bucket',
-          }),
-          {
-            description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', {
-              defaultMessage:
-                'This request counts the number of documents that fall ' +
-                'outside the criterion of the data buckets.',
-            }),
-            searchSessionId,
-          }
-        );
-
         const response = await nestedSearchSource
           .fetch$({
             abortSignal,
             sessionId: searchSessionId,
-            requestResponder,
+            inspector: {
+              adapter: inspectorRequestAdapter,
+              title: i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', {
+                defaultMessage: 'Other bucket',
+              }),
+              description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', {
+                defaultMessage:
+                  'This request counts the number of documents that fall ' +
+                  'outside the criterion of the data buckets.',
+              }),
+            },
           })
           .toPromise();
 

src/plugins/data/common/search/expressions/esaggs/request_handler.ts

@@ -40,21 +40,6 @@ export interface RequestHandlerParams {
   getNow?: () => Date;
 }
 
-function getRequestMainResponder(inspectorAdapters: Adapters, searchSessionId?: string) {
-  return inspectorAdapters.requests?.start(
-    i18n.translate('data.functions.esaggs.inspector.dataRequest.title', {
-      defaultMessage: 'Data',
-    }),
-    {
-      description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', {
-        defaultMessage:
-          'This request queries Elasticsearch to fetch the data for the visualization.',
-      }),
-      searchSessionId,
-    }
-  );
-}
-
 export const handleRequest = async ({
   abortSignal,
   aggs,
@@ -100,9 +85,7 @@ export const handleRequest = async ({
     },
   });
 
-  requestSearchSource.setField('aggs', function () {
-    return aggs.toDsl(metricsAtAllLevels);
-  });
+  requestSearchSource.setField('aggs', aggs);
 
   requestSearchSource.onRequestStart((paramSearchSource, options) => {
     return aggs.onSearchRequestStart(paramSearchSource, options);
@@ -128,31 +111,23 @@ export const handleRequest = async ({
   requestSearchSource.setField('query', query);
 
   inspectorAdapters.requests?.reset();
-  const requestResponder = getRequestMainResponder(inspectorAdapters, searchSessionId);
 
-  const response$ = await requestSearchSource.fetch$({
-    abortSignal,
-    sessionId: searchSessionId,
-    requestResponder,
-  });
-
-  // Note that rawResponse is not deeply cloned here, so downstream applications using courier
-  // must take care not to mutate it, or it could have unintended side effects, e.g. displaying
-  // response data incorrectly in the inspector.
-  let response = await response$.toPromise();
-  for (const agg of aggs.aggs) {
-    if (agg.enabled && typeof agg.type.postFlightRequest === 'function') {
-      response = await agg.type.postFlightRequest(
-        response,
-        aggs,
-        agg,
-        requestSearchSource,
-        inspectorAdapters.requests,
-        abortSignal,
-        searchSessionId
-      );
-    }
-  }
+  const response = await requestSearchSource
+    .fetch$({
+      abortSignal,
+      sessionId: searchSessionId,
+      inspector: {
+        adapter: inspectorAdapters.requests,
+        title: i18n.translate('data.functions.esaggs.inspector.dataRequest.title', {
+          defaultMessage: 'Data',
+        }),
+        description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', {
+          defaultMessage:
+            'This request queries Elasticsearch to fetch the data for the visualization.',
+        }),
+      },
+    })
+    .toPromise();
 
   const parsedTimeRange = timeRange ? calculateBounds(timeRange, { forceNow }) : null;
   const tabifyParams = {