[Reporting/CSV/7.17] allow query from state in parameters (#149297)

## Summary

In 7.17.9, we're restoring a report generation API endpoint to create
CSV reports based on saved searches. There has been one PR that serves
as the first iteration: #148030

This resolves a missing capability from the first iteration. In the
first iteration, POST bodies could allow an additional time range
filter, which is merged with any saved filters or queries stored in the
saved search object:
```
POST /api/reporting/v1/generate/csv/saved-object/search:${savedSearchId}
{
  timerange: {
    min: '2015-09-20 10:23:36.052',
    max: '2015-09-20 10:25:55.744'
  }
}
```

This PR is a second iteration. It allows additional "unsaved state" to
be merged with the saved object at the time of report generation.
```
POST /api/reporting/v1/generate/csv/saved-object/search:${savedSearchId}
 state": {
  "query": { "multi_match": { "type": "best_fields", "query": "cognac", "lenient": true } },
  "timerange": <same as before>
```
```
POST /api/reporting/v1/generate/csv/saved-object/search:${savedSearchId}
"state": {
  "query": [
    { "multi_match": { "type": "best_fields", "query": "cognac", "lenient": true } },
    { "bool": { "must_not": { "multi_match": { "type": "best_fields", "query": "Pyramidustries", "lenient": true } } } }
  ],
 timerange": <same as before>
```

### Details
In the details of #148030, it was
stated:
> Does not allow "raw state" to be merged with the Search object, as in
the previous code (from 7.3-7.8). Otherwise, the API is compatible with
the previous code.

This PR pushes a bit back against that limitation. Now, requests to
generate a report can accept a `state` field in the POST body. This
field contains additional "unsaved state" that gets merged with the
contents of the stored saved search object.

However, the entire functionality of allowing `sort` and
`docvalue_fields` keys in the request, is still not restored from the
functionality that was implemented in 7.3-7.8. This limitation exists to
minimize the complexity of the restored endpoint.

Both of the non-restored keys are related to the sorting of documents.
The sorting of documents is controlled by the saved search object only.
The user can change the sort of the CSV after downloading the report, in
a spreadsheet application or by programmatically working on the file.

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/reporting/common/types/export_types/csv_saved_object.ts

@@ -5,10 +5,14 @@
  * 2.0.
  */
 
+import { estypes } from '@elastic/elasticsearch';
 import type { BaseParams, BasePayload } from '../base';
 
 interface CsvFromSavedObjectBase {
   objectType: 'saved search';
+  state?: {
+    query?: estypes.QueryDslQueryContainer | estypes.QueryDslQueryContainer[];
+  };
   timerange?: {
     timezone?: string;
     min?: string | number;

x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.ts

@@ -91,7 +91,8 @@ export const runTaskFnFactory: RunTaskFnFactory<RunTaskFnType> = (reporting, _lo
       { uiSettings },
       await searchSourceStart.create(searchSourceFieldsWithRefs),
       savedSearch,
-      job.timerange
+      job.timerange,
+      job.state
     );
 
     const jobParamsCsv: JobParamsCSV = { ...job, columns, searchSource };

x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.test.ts

@@ -151,6 +151,49 @@ describe('get_sharing_data', () => {
     ]);
   });
 
+  it('with saved search containing a filter', async () => {
+    mockIndexPattern = createMockIndexPattern();
+    mockSearchSource.getField = jest.fn((fieldName) => {
+      if (fieldName === 'filter') {
+        return {
+          query: {
+            range: {
+              '@timestamp': { gte: '2015-09-20T10:19:40.307Z', lt: '2015-09-20T10:26:56.221Z' },
+            },
+          },
+        };
+      }
+      return mockSearchSourceGetField(fieldName);
+    });
+
+    const sharingData = await getSharingData({ uiSettings }, mockSearchSource, mockSavedSearch);
+    expect(sharingData.columns).toMatchInlineSnapshot(`
+      Array [
+        "@timestamp",
+        "clientip",
+        "extension",
+      ]
+    `);
+    expect(mockSearchSource.setField).toBeCalledTimes(3);
+    expect(mockSearchSource.setField).toHaveBeenNthCalledWith(1, 'sort', [
+      { '@timestamp': 'desc' },
+    ]);
+    expect(mockSearchSource.setField).toHaveBeenNthCalledWith(2, 'fields', [
+      '@timestamp',
+      'clientip',
+      'extension',
+    ]);
+    expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [
+      {
+        query: {
+          range: {
+            '@timestamp': { gte: '2015-09-20T10:19:40.307Z', lt: '2015-09-20T10:26:56.221Z' },
+          },
+        },
+      },
+    ]);
+  });
+
   it('with saved search containing filters', async () => {
     mockIndexPattern = createMockIndexPattern();
     mockSearchSource.getField = jest.fn((fieldName) => {
@@ -319,4 +362,88 @@ describe('get_sharing_data', () => {
       { query: { match_phrase: { 'extension.raw': 'gif' } } },
     ]);
   });
+
+  it('with saved search containing a filter from external state in job params', async () => {
+    mockIndexPattern = createMockIndexPattern();
+    const mockJobParamsUnsavedState = {
+      query: { multi_match: { type: 'best_fields' as const, query: 'cognac', lenient: true } },
+    };
+    await getSharingData(
+      { uiSettings },
+      mockSearchSource,
+      mockSavedSearch,
+      undefined,
+      mockJobParamsUnsavedState
+    );
+    expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [
+      { multi_match: { lenient: true, query: 'cognac', type: 'best_fields' } },
+    ]);
+  });
+
+  it('with saved search containing multiple filters from external state in job params', async () => {
+    mockIndexPattern = createMockIndexPattern();
+    const mockJobParamsUnsavedState = {
+      query: [
+        { multi_match: { type: 'best_fields' as const, query: 'cognac', lenient: true } },
+        {
+          bool: {
+            must_not: {
+              multi_match: { type: 'best_fields' as const, query: 'Pyramidustries', lenient: true },
+            },
+          },
+        },
+      ],
+    };
+    await getSharingData(
+      { uiSettings },
+      mockSearchSource,
+      mockSavedSearch,
+      undefined,
+      mockJobParamsUnsavedState
+    );
+    expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [
+      { multi_match: { lenient: true, query: 'cognac', type: 'best_fields' } },
+      {
+        bool: {
+          must_not: {
+            multi_match: { lenient: true, query: 'Pyramidustries', type: 'best_fields' },
+          },
+        },
+      },
+    ]);
+  });
+
+  it('with saved search containing a filter from external state in job params to be combined with a given time filter', async () => {
+    mockIndexPattern = createMockIndexPattern();
+    const mockTimeRangeFilterFromRequest = {
+      min: '2023-01-04T21:26:18.620Z',
+      max: '2023-01-05T09:21:21.543Z',
+    };
+    const mockJobParamsUnsavedState = {
+      query: [{ multi_match: { type: 'best_fields' as const, query: 'cognac', lenient: true } }],
+    };
+
+    await getSharingData(
+      { uiSettings },
+      mockSearchSource,
+      mockSavedSearch,
+      mockTimeRangeFilterFromRequest,
+      mockJobParamsUnsavedState
+    );
+    expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [
+      {
+        meta: { index: 'logstash-*' },
+        query: {
+          range: {
+            '@timestamp': {
+              format: 'strict_date_optional_time',
+              gte: '2023-01-04T21:26:18.620Z',
+              lte: '2023-01-05T09:21:21.543Z',
+            },
+          },
+        },
+      },
+      { multi_match: { lenient: true, query: 'cognac', type: 'best_fields' } },
+    ]);
+  });
 });

x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { estypes } from '@elastic/elasticsearch';
 import { Filter } from '@kbn/es-query';
 import type { IUiSettingsClient, SavedObject } from 'kibana/server';
 import moment from 'moment-timezone';
@@ -23,6 +24,17 @@ export type SavedSearchObjectType = SavedObject<
 function isStringArray(arr: unknown | string[]): arr is string[] {
   return Array.isArray(arr) && arr.every((p) => typeof p === 'string');
 }
+type FilterResponse = undefined | Filter | Filter[] | (() => Filter | Filter[] | undefined);
+function normalizeFilter(savedSearchFilterTmp?: FilterResponse) {
+  let savedSearchFilter: Filter[] | undefined;
+  if (savedSearchFilterTmp && Array.isArray(savedSearchFilterTmp)) {
+    // can not include functions: could be recursive
+    savedSearchFilter = [...savedSearchFilterTmp.filter((f) => typeof f !== 'function')];
+  } else if (savedSearchFilterTmp && typeof savedSearchFilterTmp !== 'function') {
+    savedSearchFilter = [savedSearchFilterTmp];
+  }
+  return savedSearchFilter;
+}
 
 /**
  * Partially copied from src/plugins/discover/public/application/apps/main/utils/get_sharing_data.ts
@@ -32,7 +44,10 @@ export async function getSharingData(
   services: { uiSettings: IUiSettingsClient },
   currentSearchSource: ISearchSource,
   savedSearch: SavedSearchObjectType,
-  jobParamsTimeRange?: { min?: string | number; max?: string | number; timezone?: string }
+  jobParamsTimeRange?: { min?: string | number; max?: string | number; timezone?: string },
+  jobParamsUnsavedState?: {
+    query?: estypes.QueryDslQueryContainer | estypes.QueryDslQueryContainer[];
+  }
 ) {
   const searchSource = currentSearchSource.createCopy();
   const index = searchSource.getField('index');
@@ -108,27 +123,22 @@ export async function getSharingData(
   // Combine the time range filter from the job request body with any filters that have been saved into the saved search object
   // NOTE: if the filters that were saved into the search are NOT an array, it may be a function. Function
   // filters are not supported in this API.
-  const savedSearchFilterTmp = searchSource.getField('filter');
-  searchSource.removeField('filter');
+  let combinedFilters: Filter[] = [];
+  const savedSearchFilter = normalizeFilter(searchSource.getField('filter'));
+  const jobParamsStateFilter = normalizeFilter(jobParamsUnsavedState?.query as FilterResponse);
 
-  let combinedFilters: Filter[] | undefined;
-  let savedSearchFilter: Filter[] | undefined;
-  if (savedSearchFilterTmp && Array.isArray(savedSearchFilterTmp)) {
-    // can not include functions: could be recursive
-    savedSearchFilter = [...savedSearchFilterTmp.filter((f) => typeof f !== 'function')];
-  } else if (savedSearchFilterTmp && typeof savedSearchFilterTmp !== 'function') {
-    savedSearchFilter = [savedSearchFilterTmp];
+  if (jobParamsTimeRangeFilter) {
+    combinedFilters.push(jobParamsTimeRangeFilter);
   }
-
-  if (savedSearchFilter && jobParamsTimeRangeFilter) {
-    combinedFilters = [jobParamsTimeRangeFilter, ...savedSearchFilter];
-  } else if (savedSearchFilter) {
-    combinedFilters = [...savedSearchFilter];
-  } else if (jobParamsTimeRangeFilter) {
-    combinedFilters = [jobParamsTimeRangeFilter];
+  if (savedSearchFilter && savedSearchFilter.length > 0) {
+    combinedFilters = combinedFilters.concat(savedSearchFilter);
+  }
+  if (jobParamsStateFilter && jobParamsStateFilter?.length > 0) {
+    combinedFilters = combinedFilters.concat(jobParamsStateFilter);
   }
 
-  if (combinedFilters) {
+  searchSource.removeField('filter');
+  if (combinedFilters.length > 0) {
     searchSource.setField('filter', combinedFilters);
   }
 

x-pack/plugins/reporting/server/routes/generate/generate_from_savedobject.ts

@@ -26,6 +26,27 @@ const CsvSavedSearchExportParamsSchema = schema.object({
 
 const CsvSavedSearchExportBodySchema = schema.nullable(
   schema.object({
+    state: schema.maybe(
+      schema.object({
+        query: schema.maybe(
+          schema.any({
+            validate: (input) => {
+              const failMessage = 'Must be a object of Query DSL or an array of Query DSL objects';
+              if (typeof input !== 'object') {
+                return failMessage;
+              }
+              if (Array.isArray(input)) {
+                for (let i = 0; i < input.length; i++) {
+                  if (typeof input[i] !== 'object') {
+                    return failMessage;
+                  }
+                }
+              }
+            },
+          })
+        ),
+      })
+    ),
     timerange: schema.maybe(
       schema.object({
         timezone: schema.maybe(schema.string()),
@@ -81,6 +102,7 @@ export function registerGenerateFromSavedObject(reporting: ReportingCore, logger
           browserTimezone: req.body?.timerange?.timezone || 'UTC',
           timerange: req.body?.timerange,
           savedObjectId: req.params.savedObjectId,
+          state: req.body?.state,
           title: searchObject.attributes.title ?? 'Unknown search',
           objectType: 'saved search',
           version: '7.17',

x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts

@@ -345,6 +345,50 @@ export default ({ getService }: FtrProviderContext) => {
         });
       });
 
+      describe('export with saved filters, job params timerange filter, and query from unsaved state', () => {
+        let job: ReportApiJSON;
+        let path: string;
+        let csvFile: string;
+
+        before(async () => {
+          const { text, status } = await requestCsvFromSavedSearch(
+            LOGS_SAVED_SEARCH_DATE_FILTER_ID,
+            {
+              state: {
+                query: [{ multi_match: { type: 'best_fields', query: 'css', lenient: true } }],
+              },
+              timerange: { min: '2015-09-20 10:23:36.052', max: '2015-09-20 10:25:55.744' },
+            }
+          );
+          expect(status).to.eql(200);
+          const { payload } = JSON.parse(text);
+          job = payload.job;
+          path = payload.path;
+          await reportingAPI.waitForJobToFinish(path);
+          const response = await supertest.get(path);
+          expect(response.header['content-disposition']).to.equal(
+            'inline; filename="A Saved Search with a date filter.csv"'
+          );
+          expect(response.header['content-type']).to.equal('text/csv; charset=utf-8');
+          csvFile = response.text;
+        });
+
+        it('job response data is correct', () => {
+          expect(path).to.be.a('string');
+          expect(job).to.be.an('object');
+          expect(job.attempts).equal(0);
+          expect(job.created_by).equal('elastic');
+          expect(job.jobtype).equal('csv_saved_object');
+          expect(job.payload.objectType).equal('saved search');
+          expect(job.payload.title).equal('A Saved Search with a date filter');
+          expect(job.payload.version).equal('7.17');
+        });
+
+        it('csv file matches', () => {
+          expectSnapshot(csvFile).toMatch();
+        });
+      });
+
       describe('export with no saved filters and job post params', () => {
         let job: ReportApiJSON;
         let path: string;