[OnWeek][Discover] Allow to fetch more documents on Discover page

src/plugins/discover/common/constants.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+export const MAX_LOADED_GRID_ROWS = 10000;
 export const DEFAULT_ROWS_PER_PAGE = 100;
 export const ROWS_PER_PAGE_OPTIONS = [10, 25, 50, DEFAULT_ROWS_PER_PAGE, 250, 500];
 export enum VIEW_MODE {

src/plugins/discover/common/utils/sorting/get_es_query_sort.test.ts

@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { SortDirection } from '@kbn/data-plugin/common';
+import { createStubDataView } from '@kbn/data-views-plugin/common/data_view.stub';
+import {
+  getEsQuerySort,
+  getESQuerySortForTieBreaker,
+  getESQuerySortForTimeField,
+  getTieBreakerFieldName,
+} from './get_es_query_sort';
+import { CONTEXT_TIE_BREAKER_FIELDS_SETTING } from '@kbn/discover-utils';
+import { IUiSettingsClient } from '@kbn/core-ui-settings-browser';
+
+const dataView = createStubDataView({
+  spec: {
+    id: 'logstash-*',
+    fields: {
+      test_field: {
+        name: 'test_field',
+        type: 'string',
+        esTypes: ['keyword'],
+        aggregatable: true,
+        searchable: true,
+      },
+      test_field_not_sortable: {
+        name: 'test_field_not_sortable',
+        type: 'string',
+        esTypes: ['keyword'],
+        aggregatable: false,
+        searchable: false,
+      },
+    },
+    title: 'logstash-*',
+    timeFieldName: '@timestamp',
+  },
+});
+
+describe('get_es_query_sort', function () {
+  test('getEsQuerySort should return sort params', function () {
+    expect(
+      getEsQuerySort({
+        sortDir: SortDirection.desc,
+        timeFieldName: 'testTimeField',
+        isTimeNanosBased: false,
+        tieBreakerFieldName: 'testTieBreakerField',
+      })
+    ).toStrictEqual([
+      { testTimeField: { format: 'strict_date_optional_time', order: 'desc' } },
+      { testTieBreakerField: 'desc' },
+    ]);
+
+    expect(
+      getEsQuerySort({
+        sortDir: SortDirection.asc,
+        timeFieldName: 'testTimeField',
+        isTimeNanosBased: true,
+        tieBreakerFieldName: 'testTieBreakerField',
+      })
+    ).toStrictEqual([
+      {
+        testTimeField: {
+          format: 'strict_date_optional_time_nanos',
+          numeric_type: 'date_nanos',
+          order: 'asc',
+        },
+      },
+      { testTieBreakerField: 'asc' },
+    ]);
+  });
+
+  test('getESQuerySortForTimeField should return time field as sort param', function () {
+    expect(
+      getESQuerySortForTimeField({
+        sortDir: SortDirection.desc,
+        timeFieldName: 'testTimeField',
+        isTimeNanosBased: false,
+      })
+    ).toStrictEqual({
+      testTimeField: {
+        format: 'strict_date_optional_time',
+        order: 'desc',
+      },
+    });
+
+    expect(
+      getESQuerySortForTimeField({
+        sortDir: SortDirection.asc,
+        timeFieldName: 'testTimeField',
+        isTimeNanosBased: true,
+      })
+    ).toStrictEqual({
+      testTimeField: {
+        format: 'strict_date_optional_time_nanos',
+        numeric_type: 'date_nanos',
+        order: 'asc',
+      },
+    });
+  });
+
+  test('getESQuerySortForTieBreaker should return tie breaker as sort param', function () {
+    expect(
+      getESQuerySortForTieBreaker({
+        sortDir: SortDirection.desc,
+        tieBreakerFieldName: 'testTieBreaker',
+      })
+    ).toStrictEqual({ testTieBreaker: 'desc' });
+  });
+
+  test('getTieBreakerFieldName should return a correct tie breaker', function () {
+    expect(
+      getTieBreakerFieldName(dataView, {
+        get: (key) => (key === CONTEXT_TIE_BREAKER_FIELDS_SETTING ? ['_doc'] : undefined),
+      } as IUiSettingsClient)
+    ).toBe('_doc');
+
+    expect(
+      getTieBreakerFieldName(dataView, {
+        get: (key) =>
+          key === CONTEXT_TIE_BREAKER_FIELDS_SETTING
+            ? ['test_field_not_sortable', '_doc']
+            : undefined,
+      } as IUiSettingsClient)
+    ).toBe('_doc');
+
+    expect(
+      getTieBreakerFieldName(dataView, {
+        get: (key) =>
+          key === CONTEXT_TIE_BREAKER_FIELDS_SETTING ? ['test_field', '_doc'] : undefined,
+      } as IUiSettingsClient)
+    ).toBe('test_field');
+
+    expect(
+      getTieBreakerFieldName(dataView, {
+        get: (key) => undefined,
+      } as IUiSettingsClient)
+    ).toBeUndefined();
+  });
+});

src/plugins/discover/common/utils/sorting/get_es_query_sort.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { EsQuerySortValue, SortDirection } from '@kbn/data-plugin/public';
+import type { DataView } from '@kbn/data-views-plugin/common';
+import type { IUiSettingsClient } from '@kbn/core-ui-settings-browser';
+import { CONTEXT_TIE_BREAKER_FIELDS_SETTING } from '@kbn/discover-utils';
+
+/**
+ * Returns `EsQuerySort` which is used to sort records in the ES query
+ * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html
+ * @param sortDir
+ * @param timeFieldName
+ * @param tieBreakerFieldName
+ * @param isTimeNanosBased
+ */
+export function getEsQuerySort({
+  sortDir,
+  timeFieldName,
+  tieBreakerFieldName,
+  isTimeNanosBased,
+}: {
+  sortDir: SortDirection;
+  timeFieldName: string;
+  tieBreakerFieldName: string;
+  isTimeNanosBased: boolean;
+}): [EsQuerySortValue, EsQuerySortValue] {
+  return [
+    getESQuerySortForTimeField({ sortDir, timeFieldName, isTimeNanosBased }),
+    getESQuerySortForTieBreaker({ sortDir, tieBreakerFieldName }),
+  ];
+}
+
+/**
+ * Prepares "sort" structure for a time field for next ES request
+ * @param sortDir
+ * @param timeFieldName
+ * @param isTimeNanosBased
+ */
+export function getESQuerySortForTimeField({
+  sortDir,
+  timeFieldName,
+  isTimeNanosBased,
+}: {
+  sortDir: SortDirection;
+  timeFieldName: string;
+  isTimeNanosBased: boolean;
+}): EsQuerySortValue {
+  return {
+    [timeFieldName]: {
+      order: sortDir,
+      ...(isTimeNanosBased
+        ? {
+            format: 'strict_date_optional_time_nanos',
+            numeric_type: 'date_nanos',
+          }
+        : { format: 'strict_date_optional_time' }),
+    },
+  };
+}
+
+/**
+ * Prepares "sort" structure for a tie breaker for next ES request
+ * @param sortDir
+ * @param tieBreakerFieldName
+ */
+export function getESQuerySortForTieBreaker({
+  sortDir,
+  tieBreakerFieldName,
+}: {
+  sortDir: SortDirection;
+  tieBreakerFieldName: string;
+}): EsQuerySortValue {
+  return { [tieBreakerFieldName]: sortDir };
+}
+
+/**
+ * The default tie breaker for Discover
+ */
+export const DEFAULT_TIE_BREAKER_NAME = '_doc';
+
+/**
+ * The list of field names that are allowed for sorting, but not included in
+ * data view fields.
+ */
+const META_FIELD_NAMES: string[] = ['_seq_no', '_doc', '_uid'];
+
+/**
+ * Returns a field from the intersection of the set of sortable fields in the
+ * given data view and a given set of candidate field names.
+ */
+export function getTieBreakerFieldName(
+  dataView: DataView,
+  uiSettings: Pick<IUiSettingsClient, 'get'>
+) {
+  const sortableFields = (uiSettings.get(CONTEXT_TIE_BREAKER_FIELDS_SETTING) || []).filter(
+    (fieldName: string) =>
+      META_FIELD_NAMES.includes(fieldName) ||
+      (dataView.fields.getByName(fieldName) || { sortable: false }).sortable
+  );
+  return sortableFields[0];
+}

src/plugins/discover/common/utils/sorting/get_sort_for_search_source.test.ts

@@ -6,35 +6,92 @@
  * Side Public License, v 1.
  */
 import type { SortOrder } from '@kbn/saved-search-plugin/public';
+import { SORT_DEFAULT_ORDER_SETTING } from '@kbn/discover-utils';
 import { getSortForSearchSource } from './get_sort_for_search_source';
 import { stubDataView, stubDataViewWithoutTimeField } from '@kbn/data-plugin/common/stubs';
+import { coreMock } from '@kbn/core/public/mocks';
 
 describe('getSortForSearchSource function', function () {
+  const core = coreMock.createStart();
+  const uiSettings = core.uiSettings;
+
+  const uiSettingWithAscSorting = coreMock.createStart().uiSettings;
+  jest
+    .spyOn(uiSettingWithAscSorting, 'get')
+    .mockImplementation((key) => (key === SORT_DEFAULT_ORDER_SETTING ? 'asc' : null));
+
   test('should be a function', function () {
     expect(typeof getSortForSearchSource === 'function').toBeTruthy();
   });
 
   test('should return an object to use for searchSource when columns are given', function () {
     const cols = [['bytes', 'desc']] as SortOrder[];
-    expect(getSortForSearchSource(cols, stubDataView)).toEqual([{ bytes: 'desc' }]);
-    expect(getSortForSearchSource(cols, stubDataView, 'asc')).toEqual([{ bytes: 'desc' }]);
+    expect(
+      getSortForSearchSource({
+        sort: cols,
+        dataView: stubDataView,
+        uiSettings,
+        includeTieBreaker: true,
+      })
+    ).toEqual([{ bytes: 'desc' }, { _doc: 'desc' }]);
 
-    expect(getSortForSearchSource(cols, stubDataViewWithoutTimeField)).toEqual([{ bytes: 'desc' }]);
-    expect(getSortForSearchSource(cols, stubDataViewWithoutTimeField, 'asc')).toEqual([
-      { bytes: 'desc' },
-    ]);
+    expect(
+      getSortForSearchSource({
+        sort: cols,
+        dataView: stubDataView,
+        uiSettings: uiSettingWithAscSorting,
+        includeTieBreaker: true,
+      })
+    ).toEqual([{ bytes: 'desc' }, { _doc: 'desc' }]);
+
+    expect(
+      getSortForSearchSource({
+        sort: cols,
+        dataView: stubDataView,
+        uiSettings: uiSettingWithAscSorting,
+      })
+    ).toEqual([{ bytes: 'desc' }]);
+
+    expect(
+      getSortForSearchSource({
+        sort: cols,
+        dataView: stubDataViewWithoutTimeField,
+        uiSettings,
+        includeTieBreaker: true,
+      })
+    ).toEqual([{ bytes: 'desc' }]);
+
+    expect(
+      getSortForSearchSource({
+        sort: cols,
+        dataView: stubDataViewWithoutTimeField,
+        uiSettings: uiSettingWithAscSorting,
+      })
+    ).toEqual([{ bytes: 'desc' }]);
   });
 
   test('should return an object to use for searchSource when no columns are given', function () {
     const cols = [] as SortOrder[];
-    expect(getSortForSearchSource(cols, stubDataView)).toEqual([{ _doc: 'desc' }]);
-    expect(getSortForSearchSource(cols, stubDataView, 'asc')).toEqual([{ _doc: 'asc' }]);
-
-    expect(getSortForSearchSource(cols, stubDataViewWithoutTimeField)).toEqual([
-      { _score: 'desc' },
-    ]);
-    expect(getSortForSearchSource(cols, stubDataViewWithoutTimeField, 'asc')).toEqual([
-      { _score: 'asc' },
+    expect(getSortForSearchSource({ sort: cols, dataView: stubDataView, uiSettings })).toEqual([
+      { _doc: 'desc' },
     ]);
+    expect(
+      getSortForSearchSource({
+        sort: cols,
+        dataView: stubDataView,
+        uiSettings: uiSettingWithAscSorting,
+      })
+    ).toEqual([{ _doc: 'asc' }]);
+
+    expect(
+      getSortForSearchSource({ sort: cols, dataView: stubDataViewWithoutTimeField, uiSettings })
+    ).toEqual([{ _score: 'desc' }]);
+    expect(
+      getSortForSearchSource({
+        sort: cols,
+        dataView: stubDataViewWithoutTimeField,
+        uiSettings: uiSettingWithAscSorting,
+      })
+    ).toEqual([{ _score: 'asc' }]);
   });
 });