Merge branch 'master' into moving-file-data-visualizer-UI-to-file-upl…

…oad-plugin

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md

@@ -19,6 +19,7 @@ export interface ISearchOptions
 |  [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
 |  [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
+|  [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) | <code>RequestResponder</code> |  |
 |  [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) &gt; [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md)
+
+## ISearchOptions.requestResponder property
+
+<b>Signature:</b>
+
+```typescript
+requestResponder?: RequestResponder;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md

@@ -53,7 +53,6 @@ search: {
             timeRange: import("../common").TimeRange | undefined;
         } | undefined;
     };
-    getRequestInspectorStats: typeof getRequestInspectorStats;
     getResponseInspectorStats: typeof getResponseInspectorStats;
     tabifyAggResponse: typeof tabifyAggResponse;
     tabifyGetColumns: typeof tabifyGetColumns;

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md

@@ -9,9 +9,9 @@ Returns body contents of the search request, often referred as query DSL.
 <b>Signature:</b>
 
 ```typescript
-getSearchRequestBody(): Promise<any>;
+getSearchRequestBody(): any;
 ```
 <b>Returns:</b>
 
-`Promise<any>`
+`any`
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md

@@ -19,6 +19,7 @@ export interface ISearchOptions
 |  [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
 |  [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
+|  [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) | <code>RequestResponder</code> |  |
 |  [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) &gt; [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md)
+
+## ISearchOptions.requestResponder property
+
+<b>Signature:</b>
+
+```typescript
+requestResponder?: RequestResponder;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md

@@ -36,8 +36,6 @@ search: {
         toAbsoluteDates: typeof toAbsoluteDates;
         calcAutoIntervalLessThan: typeof calcAutoIntervalLessThan;
     };
-    getRequestInspectorStats: typeof getRequestInspectorStats;
-    getResponseInspectorStats: typeof getResponseInspectorStats;
     tabifyAggResponse: typeof tabifyAggResponse;
     tabifyGetColumns: typeof tabifyGetColumns;
 }

examples/search_examples/public/search/app.tsx

@@ -204,8 +204,8 @@ export const SearchExamplesApp = ({
         });
       }
 
-      setRequest(await searchSource.getSearchRequestBody());
-      const res = await searchSource.fetch();
+      setRequest(searchSource.getSearchRequestBody());
+      const res = await searchSource.fetch$().toPromise();
       setResponse(res);
 
       const message = <EuiText>Searched {res.hits.total} documents.</EuiText>;

src/core/server/config/ensure_valid_configuration.test.ts

@@ -16,14 +16,40 @@ describe('ensureValidConfiguration', () => {
   beforeEach(() => {
     jest.clearAllMocks();
     configService = configServiceMock.create();
-    configService.getUsedPaths.mockReturnValue(Promise.resolve(['core', 'elastic']));
+
+    configService.validate.mockResolvedValue();
+    configService.getUsedPaths.mockReturnValue(Promise.resolve([]));
   });
 
-  it('returns normally when there is no unused keys', async () => {
-    configService.getUnusedPaths.mockResolvedValue([]);
+  it('returns normally when there is no unused keys and when the config validates', async () => {
     await expect(ensureValidConfiguration(configService as any)).resolves.toBeUndefined();
   });
 
+  it('throws when config validation fails', async () => {
+    configService.validate.mockImplementation(() => {
+      throw new Error('some message');
+    });
+
+    await expect(ensureValidConfiguration(configService as any)).rejects.toMatchInlineSnapshot(
+      `[Error: some message]`
+    );
+  });
+
+  it('throws a `CriticalError` with the correct processExitCode value when config validation fails', async () => {
+    expect.assertions(2);
+
+    configService.validate.mockImplementation(() => {
+      throw new Error('some message');
+    });
+
+    try {
+      await ensureValidConfiguration(configService as any);
+    } catch (e) {
+      expect(e).toBeInstanceOf(CriticalError);
+      expect(e.processExitCode).toEqual(78);
+    }
+  });
+
   it('throws when there are some unused keys', async () => {
     configService.getUnusedPaths.mockResolvedValue(['some.key', 'some.other.key']);
 
@@ -44,4 +70,18 @@ describe('ensureValidConfiguration', () => {
       expect(e.processExitCode).toEqual(64);
     }
   });
+
+  it('does not throw when all unused keys are included in the ignored paths', async () => {
+    configService.getUnusedPaths.mockResolvedValue(['dev.someDevKey', 'elastic.apm.enabled']);
+
+    await expect(ensureValidConfiguration(configService as any)).resolves.toBeUndefined();
+  });
+
+  it('throws when only some keys are included in the ignored paths', async () => {
+    configService.getUnusedPaths.mockResolvedValue(['dev.someDevKey', 'some.key']);
+
+    await expect(ensureValidConfiguration(configService as any)).rejects.toMatchInlineSnapshot(
+      `[Error: Unknown configuration key(s): "some.key". Check for spelling errors and ensure that expected plugins are installed.]`
+    );
+  });
 });

src/core/server/config/ensure_valid_configuration.ts

@@ -9,22 +9,27 @@
 import { ConfigService } from '@kbn/config';
 import { CriticalError } from '../errors';
 
+const ignoredPaths = ['dev.', 'elastic.apm.'];
+
+const invalidConfigExitCode = 78;
+const legacyInvalidConfigExitCode = 64;
+
 export async function ensureValidConfiguration(configService: ConfigService) {
-  await configService.validate();
+  try {
+    await configService.validate();
+  } catch (e) {
+    throw new CriticalError(e.message, 'InvalidConfig', invalidConfigExitCode, e);
+  }
 
-  const unusedConfigKeys = await configService.getUnusedPaths();
+  const unusedPaths = await configService.getUnusedPaths();
+  const unusedConfigKeys = unusedPaths.filter((unusedPath) => {
+    return !ignoredPaths.some((ignoredPath) => unusedPath.startsWith(ignoredPath));
+  });
 
   if (unusedConfigKeys.length > 0) {
     const message = `Unknown configuration key(s): ${unusedConfigKeys
       .map((key) => `"${key}"`)
       .join(', ')}. Check for spelling errors and ensure that expected plugins are installed.`;
-    throw new InvalidConfigurationError(message);
-  }
-}
-
-class InvalidConfigurationError extends CriticalError {
-  constructor(message: string) {
-    super(message, 'InvalidConfig', 64);
-    Object.setPrototypeOf(this, InvalidConfigurationError.prototype);
+    throw new CriticalError(message, 'InvalidConfig', legacyInvalidConfigExitCode);
   }
 }

src/core/server/server.ts

@@ -36,7 +36,6 @@ import { config as cspConfig } from './csp';
 import { config as elasticsearchConfig } from './elasticsearch';
 import { config as httpConfig } from './http';
 import { config as loggingConfig } from './logging';
-import { config as devConfig } from './dev';
 import { config as kibanaConfig } from './kibana_config';
 import { savedObjectsConfig, savedObjectsMigrationConfig } from './saved_objects';
 import { config as uiSettingsConfig } from './ui_settings';
@@ -303,7 +302,6 @@ export class Server {
       loggingConfig,
       httpConfig,
       pluginsConfig,
-      devConfig,
       kibanaConfig,
       savedObjectsConfig,
       savedObjectsMigrationConfig,

src/plugins/data/common/search/aggs/buckets/terms.ts

@@ -8,7 +8,6 @@
 
 import { noop } from 'lodash';
 import { i18n } from '@kbn/i18n';
-import type { RequestAdapter } from 'src/plugins/inspector/common';
 
 import { BucketAggType, IBucketAggConfig } from './bucket_agg_type';
 import { BUCKET_TYPES } from './bucket_agg_types';
@@ -21,7 +20,6 @@ import { aggTermsFnName } from './terms_fn';
 import { AggConfigSerialized, BaseAggParams } from '../types';
 
 import { KBN_FIELD_TYPES } from '../../../../common';
-import { getRequestInspectorStats, getResponseInspectorStats } from '../../expressions';
 
 import {
   buildOtherBucketAgg,
@@ -103,36 +101,28 @@ export const getTermsBucketAgg = () =>
 
         nestedSearchSource.setField('aggs', filterAgg);
 
-        let request: ReturnType<RequestAdapter['start']> | undefined;
-        if (inspectorRequestAdapter) {
-          request = inspectorRequestAdapter.start(
-            i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', {
-              defaultMessage: 'Other bucket',
+        const requestResponder = inspectorRequestAdapter?.start(
+          i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', {
+            defaultMessage: 'Other bucket',
+          }),
+          {
+            description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', {
+              defaultMessage:
+                'This request counts the number of documents that fall ' +
+                'outside the criterion of the data buckets.',
             }),
-            {
-              description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', {
-                defaultMessage:
-                  'This request counts the number of documents that fall ' +
-                  'outside the criterion of the data buckets.',
-              }),
-              searchSessionId,
-            }
-          );
-          nestedSearchSource.getSearchRequestBody().then((body) => {
-            request!.json(body);
-          });
-          request.stats(getRequestInspectorStats(nestedSearchSource));
-        }
+            searchSessionId,
+          }
+        );
+
+        const response = await nestedSearchSource
+          .fetch$({
+            abortSignal,
+            sessionId: searchSessionId,
+            requestResponder,
+          })
+          .toPromise();
 
-        const response = await nestedSearchSource.fetch({
-          abortSignal,
-          sessionId: searchSessionId,
-        });
-        if (request) {
-          request
-            .stats(getResponseInspectorStats(response, nestedSearchSource))
-            .ok({ json: response });
-        }
         resp = mergeOtherBucketAggResponse(aggConfigs, resp, response, aggConfig, filterAgg());
       }
       if (aggConfig.params.missingBucket) {

src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts

@@ -133,7 +133,7 @@ describe('esaggs expression function - public', () => {
   test('calls searchSource.fetch', async () => {
     await handleRequest(mockParams);
     const searchSource = await mockParams.searchSourceService.create();
-    expect(searchSource.fetch).toHaveBeenCalledWith({
+    expect(searchSource.fetch$).toHaveBeenCalledWith({
       abortSignal: mockParams.abortSignal,
       sessionId: mockParams.searchSessionId,
     });

src/plugins/data/common/search/expressions/esaggs/request_handler.ts

@@ -22,7 +22,6 @@ import {
 import { IAggConfigs } from '../../aggs';
 import { ISearchStartSearchSource } from '../../search_source';
 import { tabifyAggResponse } from '../../tabify';
-import { getRequestInspectorStats, getResponseInspectorStats } from '../utils';
 
 /** @internal */
 export interface RequestHandlerParams {
@@ -41,6 +40,21 @@ export interface RequestHandlerParams {
   getNow?: () => Date;
 }
 
+function getRequestMainResponder(inspectorAdapters: Adapters, searchSessionId?: string) {
+  return inspectorAdapters.requests?.start(
+    i18n.translate('data.functions.esaggs.inspector.dataRequest.title', {
+      defaultMessage: 'Data',
+    }),
+    {
+      description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', {
+        defaultMessage:
+          'This request queries Elasticsearch to fetch the data for the visualization.',
+      }),
+      searchSessionId,
+    }
+  );
+}
+
 export const handleRequest = async ({
   abortSignal,
   aggs,
@@ -113,52 +127,19 @@ export const handleRequest = async ({
   requestSearchSource.setField('filter', filters);
   requestSearchSource.setField('query', query);
 
-  let request;
-  if (inspectorAdapters.requests) {
-    inspectorAdapters.requests.reset();
-    request = inspectorAdapters.requests.start(
-      i18n.translate('data.functions.esaggs.inspector.dataRequest.title', {
-        defaultMessage: 'Data',
-      }),
-      {
-        description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', {
-          defaultMessage:
-            'This request queries Elasticsearch to fetch the data for the visualization.',
-        }),
-        searchSessionId,
-      }
-    );
-    request.stats(getRequestInspectorStats(requestSearchSource));
-  }
-
-  try {
-    const response = await requestSearchSource.fetch({
-      abortSignal,
-      sessionId: searchSessionId,
-    });
-
-    if (request) {
-      request.stats(getResponseInspectorStats(response, searchSource)).ok({ json: response });
-    }
+  inspectorAdapters.requests?.reset();
+  const requestResponder = getRequestMainResponder(inspectorAdapters, searchSessionId);
 
-    (searchSource as any).rawResponse = response;
-  } catch (e) {
-    // Log any error during request to the inspector
-    if (request) {
-      request.error({ json: e });
-    }
-    throw e;
-  } finally {
-    // Add the request body no matter if things went fine or not
-    if (request) {
-      request.json(await requestSearchSource.getSearchRequestBody());
-    }
-  }
+  const response$ = await requestSearchSource.fetch$({
+    abortSignal,
+    sessionId: searchSessionId,
+    requestResponder,
+  });
 
   // Note that rawResponse is not deeply cloned here, so downstream applications using courier
   // must take care not to mutate it, or it could have unintended side effects, e.g. displaying
   // response data incorrectly in the inspector.
-  let response = (searchSource as any).rawResponse;
+  let response = await response$.toPromise();
   for (const agg of aggs.aggs) {
     if (agg.enabled && typeof agg.type.postFlightRequest === 'function') {
       response = await agg.type.postFlightRequest(

src/plugins/data/common/search/expressions/utils/index.ts

@@ -6,5 +6,4 @@
  * Side Public License, v 1.
  */
 
-export * from './courier_inspector_stats';
 export * from './function_wrapper';