[Reporting] use point-in-time for paging search results (#144201)

* [Reporting] use point-in-time for paging search results

* add new PIT tests to data plugin

* fix deprecation

* update point-in-time ID to the latest one received

* add warning for shard failure

* fix/cleanup csv generation test

* add requestTimeout to openPit request

* logging polishes

* fix test

* remove confusing comment

Co-authored-by: Jean-Louis Leysens <[email protected]>

src/plugins/data/common/search/search_source/search_source.test.ts

@@ -903,6 +903,13 @@ describe('SearchSource', () => {
       expect(Object.keys(JSON.parse(searchSourceJSON))).toEqual(['highlightAll', 'from', 'sort']);
     });
 
+    test('should add pit', () => {
+      const pit = { id: 'flimflam', keep_alive: '1m' };
+      searchSource.setField('pit', pit);
+      const { searchSourceJSON } = searchSource.serialize();
+      expect(searchSourceJSON).toBe(JSON.stringify({ pit }));
+    });
+
     test('should serialize filters', () => {
       const filter = [
         {

src/plugins/data/common/search/search_source/search_source.ts

@@ -667,6 +667,8 @@ export class SearchSource {
           getConfig(UI_SETTINGS.SORT_OPTIONS)
         );
         return addToBody(key, sort);
+      case 'pit':
+        return addToRoot(key, val);
       case 'aggs':
         if ((val as unknown) instanceof AggConfigs) {
           return addToBody('aggs', val.toDsl());
@@ -768,7 +770,7 @@ export class SearchSource {
     const { getConfig } = this.dependencies;
     const searchRequest = this.mergeProps();
     searchRequest.body = searchRequest.body || {};
-    const { body, index, query, filters, highlightAll } = searchRequest;
+    const { body, index, query, filters, highlightAll, pit } = searchRequest;
     searchRequest.indexType = this.getIndexType(index);
     const metaFields = getConfig(UI_SETTINGS.META_FIELDS) ?? [];
 
@@ -911,6 +913,10 @@ export class SearchSource {
       delete searchRequest.highlightAll;
     }
 
+    if (pit) {
+      body.pit = pit;
+    }
+
     return searchRequest;
   }
 

src/plugins/data/common/search/search_source/types.ts

@@ -39,6 +39,9 @@ export interface ISearchStartSearchSource
   createEmpty: () => ISearchSource;
 }
 
+/**
+ * @deprecated use {@link estypes.SortResults} instead.
+ */
 export type EsQuerySearchAfter = [string | number, string | number];
 
 export enum SortDirection {
@@ -112,9 +115,13 @@ export interface SearchSourceFields {
    * {@link IndexPatternService}
    */
   index?: DataView;
-  searchAfter?: EsQuerySearchAfter;
   timeout?: string;
   terminate_after?: number;
+  searchAfter?: estypes.SortResults;
+  /**
+   * Allow querying to use a point-in-time ID for paging results
+   */
+  pit?: estypes.SearchPointInTimeReference;
 
   parent?: SearchSourceFields;
 }
@@ -160,7 +167,7 @@ export type SerializedSearchSourceFields = {
    * {@link IndexPatternService}
    */
   index?: string | DataViewSpec;
-  searchAfter?: EsQuerySearchAfter;
+  searchAfter?: estypes.SortResults;
   timeout?: string;
   terminate_after?: number;
 

src/plugins/data/server/search/strategies/es_search/es_search_strategy.ts

@@ -35,13 +35,17 @@ export const esSearchStrategyProvider = (
       throw new KbnServerError(`Unsupported index pattern type ${request.indexType}`, 400);
     }
 
+    const isPit = request.params?.body?.pit != null;
+
     const search = async () => {
       try {
         const config = await firstValueFrom(config$);
         // @ts-expect-error params fall back to any, but should be valid SearchRequest params
         const { terminateAfter, ...requestParams } = request.params ?? {};
+        const defaults = await getDefaultSearchParams(uiSettingsClient, { isPit });
+
         const params = {
-          ...(await getDefaultSearchParams(uiSettingsClient)),
+          ...defaults,
           ...getShardTimeout(config),
           ...(terminateAfter ? { terminate_after: terminateAfter } : {}),
           ...requestParams,

src/plugins/data/server/search/strategies/es_search/request_utils.ts

@@ -18,19 +18,29 @@ export function getShardTimeout(
 }
 
 export async function getDefaultSearchParams(
-  uiSettingsClient: Pick<IUiSettingsClient, 'get'>
+  uiSettingsClient: Pick<IUiSettingsClient, 'get'>,
+  options = { isPit: false }
 ): Promise<{
   max_concurrent_shard_requests?: number;
-  ignore_unavailable: boolean;
+  ignore_unavailable?: boolean;
   track_total_hits: boolean;
 }> {
   const maxConcurrentShardRequests = await uiSettingsClient.get<number>(
     UI_SETTINGS.COURIER_MAX_CONCURRENT_SHARD_REQUESTS
   );
-  return {
+
+  const defaults: Awaited<ReturnType<typeof getDefaultSearchParams>> = {
     max_concurrent_shard_requests:
       maxConcurrentShardRequests > 0 ? maxConcurrentShardRequests : undefined,
-    ignore_unavailable: true, // Don't fail if the index/indices don't exist
     track_total_hits: true,
   };
+
+  // If the request has a point-in-time ID attached, it can not include ignore_unavailable from {@link estypes.IndicesOptions}.
+  // ES will reject the request as that option was set when the point-in-time was created.
+  // Otherwise, this option allows search to not fail when the index/indices don't exist
+  if (!options.isPit) {
+    defaults.ignore_unavailable = true;
+  }
+
+  return defaults;
 }