From bc2300186aa334eb4d93d18bde065d7df434459e Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Tue, 24 Jan 2023 07:24:15 -0700 Subject: [PATCH] [Reporting/CSV/7.17] allow query from state in parameters (#149297) ## Summary In 7.17.9, we're restoring a report generation API endpoint to create CSV reports based on saved searches. There has been one PR that serves as the first iteration: https://github.com/elastic/kibana/pull/148030 This resolves a missing capability from the first iteration. In the first iteration, POST bodies could allow an additional time range filter, which is merged with any saved filters or queries stored in the saved search object: ``` POST /api/reporting/v1/generate/csv/saved-object/search:${savedSearchId} { timerange: { min: '2015-09-20 10:23:36.052', max: '2015-09-20 10:25:55.744' } } ``` This PR is a second iteration. It allows additional "unsaved state" to be merged with the saved object at the time of report generation. ``` POST /api/reporting/v1/generate/csv/saved-object/search:${savedSearchId} state": { "query": { "multi_match": { "type": "best_fields", "query": "cognac", "lenient": true } }, "timerange": ``` ``` POST /api/reporting/v1/generate/csv/saved-object/search:${savedSearchId} "state": { "query": [ { "multi_match": { "type": "best_fields", "query": "cognac", "lenient": true } }, { "bool": { "must_not": { "multi_match": { "type": "best_fields", "query": "Pyramidustries", "lenient": true } } } } ], timerange": ``` ### Details In the details of https://github.com/elastic/kibana/pull/148030, it was stated: > Does not allow "raw state" to be merged with the Search object, as in the previous code (from 7.3-7.8). Otherwise, the API is compatible with the previous code. This PR pushes a bit back against that limitation. Now, requests to generate a report can accept a `state` field in the POST body. This field contains additional "unsaved state" that gets merged with the contents of the stored saved search object. However, the entire functionality of allowing `sort` and `docvalue_fields` keys in the request, is still not restored from the functionality that was implemented in 7.3-7.8. This limitation exists to minimize the complexity of the restored endpoint. Both of the non-restored keys are related to the sorting of documents. The sorting of documents is controlled by the saved search object only. The user can change the sort of the CSV after downloading the report, in a spreadsheet application or by programmatically working on the file. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../types/export_types/csv_saved_object.ts | 4 + .../csv_saved_object/execute_job.ts | 3 +- .../lib/get_sharing_data.test.ts | 127 ++++++++++++++++++ .../csv_saved_object/lib/get_sharing_data.ts | 46 ++++--- .../generate/generate_from_savedobject.ts | 22 +++ .../__snapshots__/csv_saved_search.snap | 10 ++ .../csv_saved_search.ts | 44 ++++++ 7 files changed, 237 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/reporting/common/types/export_types/csv_saved_object.ts b/x-pack/plugins/reporting/common/types/export_types/csv_saved_object.ts index cddac3aab8218..a35574f9aa73b 100644 --- a/x-pack/plugins/reporting/common/types/export_types/csv_saved_object.ts +++ b/x-pack/plugins/reporting/common/types/export_types/csv_saved_object.ts @@ -5,10 +5,14 @@ * 2.0. */ +import { estypes } from '@elastic/elasticsearch'; import type { BaseParams, BasePayload } from '../base'; interface CsvFromSavedObjectBase { objectType: 'saved search'; + state?: { + query?: estypes.QueryDslQueryContainer | estypes.QueryDslQueryContainer[]; + }; timerange?: { timezone?: string; min?: string | number; diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.ts index e748d847b86f0..fc4ec883de742 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/execute_job.ts @@ -91,7 +91,8 @@ export const runTaskFnFactory: RunTaskFnFactory = (reporting, _lo { uiSettings }, await searchSourceStart.create(searchSourceFieldsWithRefs), savedSearch, - job.timerange + job.timerange, + job.state ); const jobParamsCsv: JobParamsCSV = { ...job, columns, searchSource }; diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.test.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.test.ts index 0b567fcb80cc6..df82d2cc98b23 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.test.ts @@ -151,6 +151,49 @@ describe('get_sharing_data', () => { ]); }); + it('with saved search containing a filter', async () => { + mockIndexPattern = createMockIndexPattern(); + mockSearchSource.getField = jest.fn((fieldName) => { + if (fieldName === 'filter') { + return { + query: { + range: { + '@timestamp': { gte: '2015-09-20T10:19:40.307Z', lt: '2015-09-20T10:26:56.221Z' }, + }, + }, + }; + } + return mockSearchSourceGetField(fieldName); + }); + + const sharingData = await getSharingData({ uiSettings }, mockSearchSource, mockSavedSearch); + expect(sharingData.columns).toMatchInlineSnapshot(` + Array [ + "@timestamp", + "clientip", + "extension", + ] + `); + expect(mockSearchSource.setField).toBeCalledTimes(3); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(1, 'sort', [ + { '@timestamp': 'desc' }, + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(2, 'fields', [ + '@timestamp', + 'clientip', + 'extension', + ]); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [ + { + query: { + range: { + '@timestamp': { gte: '2015-09-20T10:19:40.307Z', lt: '2015-09-20T10:26:56.221Z' }, + }, + }, + }, + ]); + }); + it('with saved search containing filters', async () => { mockIndexPattern = createMockIndexPattern(); mockSearchSource.getField = jest.fn((fieldName) => { @@ -319,4 +362,88 @@ describe('get_sharing_data', () => { { query: { match_phrase: { 'extension.raw': 'gif' } } }, ]); }); + + it('with saved search containing a filter from external state in job params', async () => { + mockIndexPattern = createMockIndexPattern(); + const mockJobParamsUnsavedState = { + query: { multi_match: { type: 'best_fields' as const, query: 'cognac', lenient: true } }, + }; + await getSharingData( + { uiSettings }, + mockSearchSource, + mockSavedSearch, + undefined, + mockJobParamsUnsavedState + ); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [ + { multi_match: { lenient: true, query: 'cognac', type: 'best_fields' } }, + ]); + }); + + it('with saved search containing multiple filters from external state in job params', async () => { + mockIndexPattern = createMockIndexPattern(); + const mockJobParamsUnsavedState = { + query: [ + { multi_match: { type: 'best_fields' as const, query: 'cognac', lenient: true } }, + { + bool: { + must_not: { + multi_match: { type: 'best_fields' as const, query: 'Pyramidustries', lenient: true }, + }, + }, + }, + ], + }; + await getSharingData( + { uiSettings }, + mockSearchSource, + mockSavedSearch, + undefined, + mockJobParamsUnsavedState + ); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [ + { multi_match: { lenient: true, query: 'cognac', type: 'best_fields' } }, + { + bool: { + must_not: { + multi_match: { lenient: true, query: 'Pyramidustries', type: 'best_fields' }, + }, + }, + }, + ]); + }); + + it('with saved search containing a filter from external state in job params to be combined with a given time filter', async () => { + mockIndexPattern = createMockIndexPattern(); + const mockTimeRangeFilterFromRequest = { + min: '2023-01-04T21:26:18.620Z', + max: '2023-01-05T09:21:21.543Z', + }; + const mockJobParamsUnsavedState = { + query: [{ multi_match: { type: 'best_fields' as const, query: 'cognac', lenient: true } }], + }; + + await getSharingData( + { uiSettings }, + mockSearchSource, + mockSavedSearch, + mockTimeRangeFilterFromRequest, + mockJobParamsUnsavedState + ); + expect(mockSearchSource.setField).toHaveBeenNthCalledWith(3, 'filter', [ + { + meta: { index: 'logstash-*' }, + query: { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2023-01-04T21:26:18.620Z', + lte: '2023-01-05T09:21:21.543Z', + }, + }, + }, + }, + { multi_match: { lenient: true, query: 'cognac', type: 'best_fields' } }, + ]); + }); }); diff --git a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.ts b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.ts index 979c43b150fa1..1a42014d52c0e 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_saved_object/lib/get_sharing_data.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { estypes } from '@elastic/elasticsearch'; import { Filter } from '@kbn/es-query'; import type { IUiSettingsClient, SavedObject } from 'kibana/server'; import moment from 'moment-timezone'; @@ -23,6 +24,17 @@ export type SavedSearchObjectType = SavedObject< function isStringArray(arr: unknown | string[]): arr is string[] { return Array.isArray(arr) && arr.every((p) => typeof p === 'string'); } +type FilterResponse = undefined | Filter | Filter[] | (() => Filter | Filter[] | undefined); +function normalizeFilter(savedSearchFilterTmp?: FilterResponse) { + let savedSearchFilter: Filter[] | undefined; + if (savedSearchFilterTmp && Array.isArray(savedSearchFilterTmp)) { + // can not include functions: could be recursive + savedSearchFilter = [...savedSearchFilterTmp.filter((f) => typeof f !== 'function')]; + } else if (savedSearchFilterTmp && typeof savedSearchFilterTmp !== 'function') { + savedSearchFilter = [savedSearchFilterTmp]; + } + return savedSearchFilter; +} /** * Partially copied from src/plugins/discover/public/application/apps/main/utils/get_sharing_data.ts @@ -32,7 +44,10 @@ export async function getSharingData( services: { uiSettings: IUiSettingsClient }, currentSearchSource: ISearchSource, savedSearch: SavedSearchObjectType, - jobParamsTimeRange?: { min?: string | number; max?: string | number; timezone?: string } + jobParamsTimeRange?: { min?: string | number; max?: string | number; timezone?: string }, + jobParamsUnsavedState?: { + query?: estypes.QueryDslQueryContainer | estypes.QueryDslQueryContainer[]; + } ) { const searchSource = currentSearchSource.createCopy(); const index = searchSource.getField('index'); @@ -108,27 +123,22 @@ export async function getSharingData( // Combine the time range filter from the job request body with any filters that have been saved into the saved search object // NOTE: if the filters that were saved into the search are NOT an array, it may be a function. Function // filters are not supported in this API. - const savedSearchFilterTmp = searchSource.getField('filter'); - searchSource.removeField('filter'); + let combinedFilters: Filter[] = []; + const savedSearchFilter = normalizeFilter(searchSource.getField('filter')); + const jobParamsStateFilter = normalizeFilter(jobParamsUnsavedState?.query as FilterResponse); - let combinedFilters: Filter[] | undefined; - let savedSearchFilter: Filter[] | undefined; - if (savedSearchFilterTmp && Array.isArray(savedSearchFilterTmp)) { - // can not include functions: could be recursive - savedSearchFilter = [...savedSearchFilterTmp.filter((f) => typeof f !== 'function')]; - } else if (savedSearchFilterTmp && typeof savedSearchFilterTmp !== 'function') { - savedSearchFilter = [savedSearchFilterTmp]; + if (jobParamsTimeRangeFilter) { + combinedFilters.push(jobParamsTimeRangeFilter); } - - if (savedSearchFilter && jobParamsTimeRangeFilter) { - combinedFilters = [jobParamsTimeRangeFilter, ...savedSearchFilter]; - } else if (savedSearchFilter) { - combinedFilters = [...savedSearchFilter]; - } else if (jobParamsTimeRangeFilter) { - combinedFilters = [jobParamsTimeRangeFilter]; + if (savedSearchFilter && savedSearchFilter.length > 0) { + combinedFilters = combinedFilters.concat(savedSearchFilter); + } + if (jobParamsStateFilter && jobParamsStateFilter?.length > 0) { + combinedFilters = combinedFilters.concat(jobParamsStateFilter); } - if (combinedFilters) { + searchSource.removeField('filter'); + if (combinedFilters.length > 0) { searchSource.setField('filter', combinedFilters); } diff --git a/x-pack/plugins/reporting/server/routes/generate/generate_from_savedobject.ts b/x-pack/plugins/reporting/server/routes/generate/generate_from_savedobject.ts index 6d9a9ca81d358..16f50e70c362b 100644 --- a/x-pack/plugins/reporting/server/routes/generate/generate_from_savedobject.ts +++ b/x-pack/plugins/reporting/server/routes/generate/generate_from_savedobject.ts @@ -26,6 +26,27 @@ const CsvSavedSearchExportParamsSchema = schema.object({ const CsvSavedSearchExportBodySchema = schema.nullable( schema.object({ + state: schema.maybe( + schema.object({ + query: schema.maybe( + schema.any({ + validate: (input) => { + const failMessage = 'Must be a object of Query DSL or an array of Query DSL objects'; + if (typeof input !== 'object') { + return failMessage; + } + if (Array.isArray(input)) { + for (let i = 0; i < input.length; i++) { + if (typeof input[i] !== 'object') { + return failMessage; + } + } + } + }, + }) + ), + }) + ), timerange: schema.maybe( schema.object({ timezone: schema.maybe(schema.string()), @@ -81,6 +102,7 @@ export function registerGenerateFromSavedObject(reporting: ReportingCore, logger browserTimezone: req.body?.timerange?.timezone || 'UTC', timerange: req.body?.timerange, savedObjectId: req.params.savedObjectId, + state: req.body?.state, title: searchObject.attributes.title ?? 'Unknown search', objectType: 'saved search', version: '7.17', diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_saved_search.snap b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_saved_search.snap index 6cee3c66abadb..805f0846f9bbc 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_saved_search.snap +++ b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_saved_search.snap @@ -702,3 +702,13 @@ exports[`Reporting APIs CSV Generation from Saved Search ID export from timebase 2015-09-20 10:23:36.052,51.105.100.214,jpg " `; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view export with saved filters, job params timerange filter, and query from unsaved state csv file matches 1`] = ` +"@timestamp,clientip,extension +2015-09-20 10:25:55.744,116.126.47.226,css +2015-09-20 10:25:52.360,74.224.77.232,css +2015-09-20 10:25:49.913,97.83.96.39,css +2015-09-20 10:25:44.979,175.188.44.145,css +2015-09-20 10:25:36.331,231.169.195.137,css +" +`; diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts b/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts index 567409e1ab3c5..a41a3b8a7cf3c 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts @@ -345,6 +345,50 @@ export default ({ getService }: FtrProviderContext) => { }); }); + describe('export with saved filters, job params timerange filter, and query from unsaved state', () => { + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + const { text, status } = await requestCsvFromSavedSearch( + LOGS_SAVED_SEARCH_DATE_FILTER_ID, + { + state: { + query: [{ multi_match: { type: 'best_fields', query: 'css', lenient: true } }], + }, + timerange: { min: '2015-09-20 10:23:36.052', max: '2015-09-20 10:25:55.744' }, + } + ); + expect(status).to.eql(200); + const { payload } = JSON.parse(text); + job = payload.job; + path = payload.path; + await reportingAPI.waitForJobToFinish(path); + const response = await supertest.get(path); + expect(response.header['content-disposition']).to.equal( + 'inline; filename="A Saved Search with a date filter.csv"' + ); + expect(response.header['content-type']).to.equal('text/csv; charset=utf-8'); + csvFile = response.text; + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expect(job.created_by).equal('elastic'); + expect(job.jobtype).equal('csv_saved_object'); + expect(job.payload.objectType).equal('saved search'); + expect(job.payload.title).equal('A Saved Search with a date filter'); + expect(job.payload.version).equal('7.17'); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + describe('export with no saved filters and job post params', () => { let job: ReportApiJSON; let path: string;